Add gzip compression transform and compress command line prog.

10 files changed, 261 insertions, 76 deletions

diff --git a/doc/relnotes/1_11_10.rst b/doc/relnotes/1_11_10.rst
index da382efca..58631e6f0 100644
--- a/doc/relnotes/1_11_10.rst
+++ b/doc/relnotes/1_11_10.rst
@@ -25,6 +25,11 @@ Version 1.11.10, Not Yet Released
 * Add a TLS policy hook to disable putting the value of the local clock in hello
   random fields.
 
+* All compression operations previously available as Filters are now
+  performed via the Transformation API, which minimizes memory copies.
+
+* The zlib module now also supports gzip compression and decompression.
+
 * Avoid a crash in low-entropy situations when reading from /dev/random, when
   select indicated the device was readable but by the time we start the read the
   entropy pool had been depleted.
diff --git a/src/cmd/compress.cpp b/src/cmd/compress.cpp
new file mode 100644
index 000000000..fc1ece7bd
--- /dev/null
+++ b/src/cmd/compress.cpp
@@ -0,0 +1,79 @@
+
+#include "apps.h"
+
+#include <botan/compression.h>
+#include <fstream>
+
+namespace {
+
+void do_compress(Transformation& comp, std::ifstream& in, std::ostream& out)
+   {
+   secure_vector<byte> buf;
+
+   comp.start();
+
+   while(in.good())
+      {
+      buf.resize(16*1024);
+      in.read(reinterpret_cast<char*>(&buf[0]), buf.size());
+      buf.resize(in.gcount());
+
+      comp.update(buf);
+
+      out.write(reinterpret_cast<const char*>(&buf[0]), buf.size());
+      }
+
+   buf.clear();
+   comp.finish(buf);
+   out.write(reinterpret_cast<const char*>(&buf[0]), buf.size());
+   }
+
+int compress(int argc, char* argv[])
+   {
+   const std::string in_file = argv[1];
+   std::ifstream in(in_file.c_str());
+
+   if(!in.good())
+      {
+      std::cout << "Couldn't read " << in_file << "\n";
+      return 1;
+      }
+
+   const size_t level = 6;
+   const std::string suffix = "gz";
+
+   std::unique_ptr<Transformation> compress(make_compressor(suffix, level));
+
+   const std::string out_file = in_file + "." + suffix;
+   std::ofstream out(out_file.c_str());
+
+   do_compress(*compress, in, out);
+
+   return 0;
+   }
+
+int uncompress(int argc, char* argv[])
+   {
+   const std::string in_file = argv[1];
+   std::ifstream in(in_file.c_str());
+
+   if(!in.good())
+      {
+      std::cout << "Couldn't read " << argv[1] << "\n";
+      return 1;
+      }
+
+   std::ofstream out("out");
+   const std::string suffix = "gz";
+
+   std::unique_ptr<Transformation> decompress(make_decompressor(suffix));
+
+   do_compress(*decompress, in, out);
+
+   return 0;
+   }
+
+REGISTER_APP(compress);
+REGISTER_APP(uncompress);
+
+}
diff --git a/src/cmd/main.cpp b/src/cmd/main.cpp
index 65530b23a..42acf99e7 100644
--- a/src/cmd/main.cpp
+++ b/src/cmd/main.cpp
@@ -170,17 +170,17 @@ int main(int argc, char* argv[])
       if(apps.has(cmd))
          return apps.run(cmd, argc - 1, argv + 1);
 
-      std::cout << "Unknown command " << cmd << "\n";
+      std::cerr << "Unknown command " << cmd << std::endl;
       return help(argc, argv);
       }
    catch(std::exception& e)
       {
-      std::cerr << "Exception: " << e.what() << std::endl;
+      std::cerr << e.what() << std::endl;
       return 1;
       }
    catch(...)
       {
-      std::cerr << "Unknown (...) exception caught" << std::endl;
+      std::cerr << "Unknown exception caught" << std::endl;
       return 1;
       }
 
diff --git a/src/lib/compression/bzip2/bzip2.cpp b/src/lib/compression/bzip2/bzip2.cpp
index 440c21477..313b1ed46 100644
--- a/src/lib/compression/bzip2/bzip2.cpp
+++ b/src/lib/compression/bzip2/bzip2.cpp
@@ -84,7 +84,7 @@ class Bzip_Decompression_Stream : public Bzip_Stream
          BZ2_bzDecompressEnd(streamp());
          }
 
-      bool run(u32bit flags) override
+      bool run(u32bit) override
          {
          int rc = BZ2_bzDecompress(streamp());
 
diff --git a/src/lib/compression/compression.cpp b/src/lib/compression/compression.cpp
index 3eb26e81b..35b2a4bc0 100644
--- a/src/lib/compression/compression.cpp
+++ b/src/lib/compression/compression.cpp
@@ -7,8 +7,68 @@
 
 #include <botan/compression.h>
 
+#if defined(BOTAN_HAS_ZLIB_TRANSFORM)
+  #include <botan/zlib.h>
+#endif
+
+#if defined(BOTAN_HAS_BZIP_TRANSFORM)
+  #include <botan/bzip.h>
+#endif
+
+#if defined(BOTAN_HAS_LZMA_TRANSFORM)
+  #include <botan/lzma.h>
+#endif
+
 namespace Botan {
 
+Compressor_Transformation* make_compressor(const std::string& type, size_t level)
+   {
+#if defined(BOTAN_HAS_ZLIB_TRANSFORM)
+   if(type == "zlib")
+      return new Zlib_Compression(level, false);
+   if(type == "deflate")
+      return new Zlib_Compression(level, true);
+   if(type == "gzip" || type == "gz")
+      return new Gzip_Compression(level);
+#endif
+
+#if defined(BOTAN_HAS_BZIP_TRANSFORM)
+   if(type == "bzip2" || type == "bz2")
+      return new Bzip_Compression(level);
+#endif
+
+#if defined(BOTAN_HAS_LZMA_TRANSFORM)
+   if(type == "lzma" || type == "xz")
+      return new LZMA_Compression(level);
+#endif
+
+   throw std::runtime_error("Unknown compression type " + type);
+   }
+
+Compressor_Transformation* make_decompressor(const std::string& type)
+   {
+#if defined(BOTAN_HAS_ZLIB_TRANSFORM)
+   if(type == "zlib")
+      return new Zlib_Decompression(false);
+   if(type == "deflate")
+      return new Zlib_Decompression(true);
+   if(type == "gzip" || type == "gz")
+      return new Gzip_Decompression;
+#endif
+
+#if defined(BOTAN_HAS_BZIP_TRANSFORM)
+   if(type == "bzip2" || type == "bz2")
+      return new Bzip_Decompression;
+#endif
+
+#if defined(BOTAN_HAS_LZMA_TRANSFORM)
+   if(type == "lzma" || type == "xz")
+      return new LZMA_Decompression;
+#endif
+
+   throw std::runtime_error("Unknown compression type " + type);
+   }
+
 void Stream_Compression::clear()
    {
    m_stream.reset();
@@ -19,7 +79,6 @@ secure_vector<byte> Stream_Compression::start_raw(const byte[], size_t nonce_len
    if(!valid_nonce_length(nonce_len))
       throw Invalid_IV_Length(name(), nonce_len);
 
-   clear();
    m_stream.reset(make_stream());
    return secure_vector<byte>();
    }
@@ -37,7 +96,7 @@ void Stream_Compression::process(secure_vector<byte>& buf, size_t offset, u32bit
 
    while(true)
       {
-      const bool end = m_stream->run(flags);
+      m_stream->run(flags);
 
       if(m_stream->avail_out() == 0)
          {
@@ -82,7 +141,6 @@ secure_vector<byte> Stream_Decompression::start_raw(const byte[], size_t nonce_l
    if(!valid_nonce_length(nonce_len))
       throw Invalid_IV_Length(name(), nonce_len);
 
-   clear();
    m_stream.reset(make_stream());
 
    return secure_vector<byte>();
diff --git a/src/lib/compression/compression.h b/src/lib/compression/compression.h
index 10c7f1a64..75a100cc6 100644
--- a/src/lib/compression/compression.h
+++ b/src/lib/compression/compression.h
@@ -32,6 +32,9 @@ class BOTAN_DLL Compressor_Transformation : public Transformation
          }
    };
 
+BOTAN_DLL Compressor_Transformation* make_compressor(const std::string& type, size_t level);
+BOTAN_DLL Compressor_Transformation* make_decompressor(const std::string& type);
+
 class Compression_Stream
    {
    public:
diff --git a/src/lib/compression/zlib/zlib.cpp b/src/lib/compression/zlib/zlib.cpp
index b18621640..c8f8757b1 100644
--- a/src/lib/compression/zlib/zlib.cpp
+++ b/src/lib/compression/zlib/zlib.cpp
@@ -9,6 +9,7 @@
 
 #include <botan/zlib.h>
 #include <botan/internal/comp_util.h>
+#include <ctime>
 #include <zlib.h>
 
 namespace Botan {
@@ -28,16 +29,24 @@ class Zlib_Stream : public Zlib_Style_Stream<z_stream, Bytef>
       u32bit run_flag() const override { return Z_NO_FLUSH; }
       u32bit flush_flag() const override { return Z_FULL_FLUSH; }
       u32bit finish_flag() const override { return Z_FINISH; }
+
+      int compute_window_bits(int wbits, int wbits_offset) const
+         {
+         if(wbits_offset == -1)
+            return -wbits;
+         else
+            return wbits + wbits_offset;
+         }
    };
 
 class Zlib_Compression_Stream : public Zlib_Stream
    {
    public:
-      Zlib_Compression_Stream(size_t level, bool raw_deflate)
+      Zlib_Compression_Stream(size_t level, int wbits, int wbits_offset = 0)
          {
-         // FIXME: allow specifiying memLevel and strategy
          int rc = deflateInit2(streamp(), level, Z_DEFLATED,
-                               (raw_deflate ? -15 : 15), 8, Z_DEFAULT_STRATEGY);
+                               compute_window_bits(wbits, wbits_offset),
+                               8, Z_DEFAULT_STRATEGY);
          if(rc != Z_OK)
             throw std::runtime_error("zlib deflate initialization failed");
          }
@@ -63,9 +72,9 @@ class Zlib_Compression_Stream : public Zlib_Stream
 class Zlib_Decompression_Stream : public Zlib_Stream
    {
    public:
-      Zlib_Decompression_Stream(bool raw_deflate)
+      Zlib_Decompression_Stream(int wbits, int wbits_offset = 0)
          {
-         int rc = inflateInit2(streamp(), (raw_deflate ? -15 : 15));
+         int rc = inflateInit2(streamp(), compute_window_bits(wbits, wbits_offset));
 
          if(rc == Z_MEM_ERROR)
             throw std::bad_alloc();
@@ -91,16 +100,70 @@ class Zlib_Decompression_Stream : public Zlib_Stream
          }
    };
 
+class Deflate_Compression_Stream : public Zlib_Compression_Stream
+   {
+   public:
+      Deflate_Compression_Stream(size_t level, int wbits) :
+         Zlib_Compression_Stream(level, wbits, -1) {}
+   };
+
+class Deflate_Decompression_Stream : public Zlib_Decompression_Stream
+   {
+   public:
+      Deflate_Decompression_Stream(int wbits) : Zlib_Decompression_Stream(wbits, -1) {}
+   };
+
+class Gzip_Compression_Stream : public Zlib_Compression_Stream
+   {
+   public:
+      Gzip_Compression_Stream(size_t level, int wbits, byte os_code) :
+         Zlib_Compression_Stream(level, wbits, 16)
+         {
+         std::memset(&m_header, 0, sizeof(m_header));
+         m_header.os = os_code;
+         m_header.time = std::time(0);
+
+         int rc = deflateSetHeader(streamp(), &m_header);
+         if(rc != Z_OK)
+            throw std::runtime_error("setting gzip header failed");
+         }
+
+   private:
+      ::gz_header m_header;
+   };
+
+class Gzip_Decompression_Stream : public Zlib_Decompression_Stream
+   {
+   public:
+      Gzip_Decompression_Stream(int wbits) : Zlib_Decompression_Stream(wbits, 16) {}
+   };
+
 }
 
 Compression_Stream* Zlib_Compression::make_stream() const
    {
-   return new Zlib_Compression_Stream(m_level, m_raw_deflate);
+   if(m_raw_deflate)
+      return new Deflate_Compression_Stream(m_level, 15);
+   else
+      return new Zlib_Compression_Stream(m_level, 15);
    }
 
 Compression_Stream* Zlib_Decompression::make_stream() const
    {
-   return new Zlib_Decompression_Stream(m_raw_deflate);
+   if(m_raw_deflate)
+      return new Deflate_Decompression_Stream(15);
+   else
+      return new Zlib_Decompression_Stream(15);
+   }
+
+Compression_Stream* Gzip_Compression::make_stream() const
+   {
+   return new Gzip_Compression_Stream(m_level, 15, m_os_code);
+   }
+
+Compression_Stream* Gzip_Decompression::make_stream() const
+   {
+   return new Gzip_Decompression_Stream(15);
    }
 
 }
diff --git a/src/lib/compression/zlib/zlib.h b/src/lib/compression/zlib/zlib.h
index 55da47a0d..5805efb57 100644
--- a/src/lib/compression/zlib/zlib.h
+++ b/src/lib/compression/zlib/zlib.h
@@ -46,7 +46,7 @@ class BOTAN_DLL Zlib_Decompression : public Stream_Decompression
       /**
       * @param raw_deflate if true no zlib header/trailer will be used
       */
-      Zlib_Decompression( bool raw_deflate = false) : m_raw_deflate(raw_deflate) {}
+      Zlib_Decompression(bool raw_deflate = false) : m_raw_deflate(raw_deflate) {}
 
       std::string name() const override { return "Zlib_Decompression"; }
 
@@ -56,6 +56,41 @@ class BOTAN_DLL Zlib_Decompression : public Stream_Decompression
       const bool m_raw_deflate;
    };
 
+/**
+* Gzip Compression
+*/
+class BOTAN_DLL Gzip_Compression : public Stream_Compression
+   {
+   public:
+      /**
+      * @param level how much effort to use on compressing (0 to 9);
+      *        higher levels are slower but tend to give better
+      *        compression
+      */
+      Gzip_Compression(size_t level = 6, byte os_code = 255) :
+         m_level(level), m_os_code(os_code) {}
+
+      std::string name() const override { return "Gzip_Compression"; }
+
+   private:
+      Compression_Stream* make_stream() const;
+
+      const size_t m_level;
+      const byte m_os_code;
+   };
+
+/**
+* Gzip Decompression
+*/
+class BOTAN_DLL Gzip_Decompression : public Stream_Compression
+   {
+   public:
+      std::string name() const override { return "Gzip_Decompression"; }
+
+   private:
+      Compression_Stream* make_stream() const;
+   };
+
 }
 
 #endif
diff --git a/src/lib/filters/comp_filter.cpp b/src/lib/filters/comp_filter.cpp
index 9c27978bc..a02a78eb7 100644
--- a/src/lib/filters/comp_filter.cpp
+++ b/src/lib/filters/comp_filter.cpp
@@ -6,69 +6,10 @@
 */
 
 #include <botan/comp_filter.h>
-
-#if defined(BOTAN_HAS_ZLIB_TRANSFORM)
-  #include <botan/zlib.h>
-#endif
-
-#if defined(BOTAN_HAS_BZIP_TRANSFORM)
-  #include <botan/bzip.h>
-#endif
-
-#if defined(BOTAN_HAS_LZMA_TRANSFORM)
-  #include <botan/lzma.h>
-#endif
+#include <botan/compression.h>
 
 namespace Botan {
 
-namespace {
-
-Compressor_Transformation* make_compressor(const std::string& type, size_t level)
-   {
-#if defined(BOTAN_HAS_ZLIB_TRANSFORM)
-   if(type == "zlib")
-      return new Zlib_Compression(level, false);
-   if(type == "deflate")
-      return new Zlib_Compression(level, true);
-#endif
-
-#if defined(BOTAN_HAS_BZIP_TRANSFORM)
-   if(type == "bzip2")
-      return new Bzip_Compression(level);
-#endif
-
-#if defined(BOTAN_HAS_LZMA_TRANSFORM)
-   if(type == "lzma")
-      return new LZMA_Compression(level);
-#endif
-
-   throw std::runtime_error("Unknown compression type " + type);
-   }
-
-Compressor_Transformation* make_decompressor(const std::string& type)
-   {
-#if defined(BOTAN_HAS_ZLIB_TRANSFORM)
-   if(type == "zlib")
-      return new Zlib_Decompression(false);
-   if(type == "deflate")
-      return new Zlib_Decompression(true);
-#endif
-
-#if defined(BOTAN_HAS_BZIP_TRANSFORM)
-   if(type == "bzip2")
-      return new Bzip_Decompression;
-#endif
-
-#if defined(BOTAN_HAS_LZMA_TRANSFORM)
-   if(type == "lzma")
-      return new LZMA_Decompression;
-#endif
-
-   throw std::runtime_error("Unknown compression type " + type);
-   }
-
-}
-
 Compression_Filter::Compression_Filter(const std::string& type, size_t level) :
    Compression_Decompression_Filter(make_compressor(type, level))
    {
diff --git a/src/lib/filters/comp_filter.h b/src/lib/filters/comp_filter.h
index a46932bed..863d4ffef 100644
--- a/src/lib/filters/comp_filter.h
+++ b/src/lib/filters/comp_filter.h
@@ -9,10 +9,11 @@
 #define BOTAN_COMPRESSION_FILTER_H__
 
 #include <botan/filter.h>
-#include <botan/compression.h>
 
 namespace Botan {
 
+class Compressor_Transformation;
+
 /**
 * Filter interface for compression/decompression
 */