ui-snapshot: add support for zstd compression

This patch adds support for zstd [0] compressed snapshots (*.tar.zst).
We enable multiple working threads (-T0), but keep default compression
level. The latter can be influenced by environment variable.

[0] https://www.zstd.net/

Signed-off-by: Christian Hesse <mail@eworm.de>

diff --git a/cgitrc.5.txt b/cgitrc.5.txt
index 4ad3e6444613b0592f8133f0e2196af61c8eac4b..33a6a8c0c75832645a0d1c5b9b2b65d4d8cf2466 100644 (file)
--- a/cgitrc.5.txt
+++ b/cgitrc.5.txt
@@ -407,9 +407,12 @@ side-by-side-diffs::
 snapshots::
        Text which specifies the default set of snapshot formats that cgit
        generates links for. The value is a space-separated list of zero or
-       more of the values "tar", "tar.gz", "tar.bz2", "tar.xz", "tar.lz" and
-       "zip". The special value "all" enables all snapshot formats.
-       Default value: none.
+       more of the values "tar", "tar.gz", "tar.bz2", "tar.lz", "tar.xz",
+       "tar.zst" and "zip". The special value "all" enables all snapshot
+       formats. Default value: none.
+       All compressors use default settings. Some settings can be influenced
+       with environment variables, for example set ZSTD_CLEVEL=10 in web
+       server environment for higher (but slower) zstd compression.
 
 source-filter::
        Specifies a command which will be invoked to format plaintext blobs

diff --git a/tests/setup.sh b/tests/setup.sh
index 334cca606da52c0f9828f8a0b315d2829f029392..5879348e8d4b89c06d969f6380dd18661a96e4a7 100755 (executable)
--- a/tests/setup.sh
+++ b/tests/setup.sh
@@ -104,7 +104,7 @@ virtual-root=/
 cache-root=$PWD/cache
 
 cache-size=1021
-snapshots=tar.gz tar.bz tar.lz tar.xz zip
+snapshots=tar.gz tar.bz tar.lz tar.xz tar.zst zip
 enable-log-filecount=1
 enable-log-linecount=1
 summary-log=5

diff --git a/tests/t0107-snapshot.sh b/tests/t0107-snapshot.sh
index 84995d1cfd9e73e6e36fb523d746845e64a86c79..c164d3e2572c4400962f1e14940381fed2d23eaa 100755 (executable)
--- a/tests/t0107-snapshot.sh
+++ b/tests/t0107-snapshot.sh
@@ -122,6 +122,48 @@ test_expect_success XZ 'verify untarred file-5' '
        test_line_count = 1 master/file-5
 '
 
+if test -n "$(which zstd 2>/dev/null)"; then
+       test_set_prereq ZSTD
+else
+       say 'Skipping ZSTD validation tests: zstd not found'
+fi
+
+test_expect_success ZSTD 'get foo/snapshot/master.tar.zst' '
+       cgit_url "foo/snapshot/master.tar.zst" >tmp
+'
+
+test_expect_success ZSTD 'check html headers' '
+       head -n 1 tmp |
+       grep "Content-Type: application/x-zstd" &&
+
+       head -n 2 tmp |
+       grep "Content-Disposition: inline; filename=.master.tar.zst."
+'
+
+test_expect_success ZSTD 'strip off the header lines' '
+       strip_headers <tmp >master.tar.zst
+'
+
+test_expect_success ZSTD 'verify zstd format' '
+       zstd --test master.tar.zst &&
+       cp master.tar.zst /tmp/.
+'
+
+test_expect_success ZSTD 'untar' '
+       rm -rf master &&
+       tar --zstd -xf master.tar.zst
+'
+
+test_expect_success ZSTD 'count files' '
+       ls master/ >output &&
+       test_line_count = 5 output
+'
+
+test_expect_success ZSTD 'verify untarred file-5' '
+       grep "^5$" master/file-5 &&
+       test_line_count = 1 master/file-5
+'
+
 test_expect_success 'get foo/snapshot/master.zip' '
        cgit_url "foo/snapshot/master.zip" >tmp
 '

diff --git a/ui-snapshot.c b/ui-snapshot.c
index 92cde42fbf0d809d1385a6455fc99a8fc7eedde0..556d3ed4125aac62a6d307d10b8869cc544d6118 100644 (file)
--- a/ui-snapshot.c
+++ b/ui-snapshot.c
@@ -91,6 +91,12 @@ static int write_tar_xz_archive(const char *hex, const char *prefix)
        return write_compressed_tar_archive(hex, prefix, argv);
 }
 
+static int write_tar_zstd_archive(const char *hex, const char *prefix)
+{
+       char *argv[] = { "zstd", "-T0", NULL };
+       return write_compressed_tar_archive(hex, prefix, argv);
+}
+
 const struct cgit_snapshot_format cgit_snapshot_formats[] = {
        /* .tar must remain the 0 index */
        { ".tar",       "application/x-tar",    write_tar_archive       },
@@ -98,6 +104,7 @@ const struct cgit_snapshot_format cgit_snapshot_formats[] = {
        { ".tar.bz2",   "application/x-bzip2",  write_tar_bzip2_archive },
        { ".tar.lz",    "application/x-lzip",   write_tar_lzip_archive  },
        { ".tar.xz",    "application/x-xz",     write_tar_xz_archive    },
+       { ".tar.zst",   "application/x-zstd",   write_tar_zstd_archive  },
        { ".zip",       "application/x-zip",    write_zip_archive       },
        { NULL }
 };