Convert compression filters to in-place transforms and refactor

to minimize the amount of logic needed in the files specific to each
library.

26 files changed, 1061 insertions, 1193 deletions

diff --git a/src/lib/compression/bzip2/bzip2.cpp b/src/lib/compression/bzip2/bzip2.cpp
new file mode 100644
index 000000000..440c21477
--- /dev/null
+++ b/src/lib/compression/bzip2/bzip2.cpp
@@ -0,0 +1,112 @@
+/*
+* Bzip Compressor
+* (C) 2001 Peter J Jones
+*     2001-2007,2014 Jack Lloyd
+*     2006 Matt Johnston
+*
+* Distributed under the terms of the Botan license
+*/
+
+#include <botan/bzip2.h>
+#include <botan/internal/comp_util.h>
+
+#define BZ_NO_STDIO
+#include <bzlib.h>
+
+namespace Botan {
+
+namespace {
+
+class Bzip_Stream : public Zlib_Style_Stream<bz_stream, char>
+   {
+   public:
+      Bzip_Stream()
+         {
+         streamp()->opaque = alloc();
+         streamp()->bzalloc = Compression_Alloc_Info::malloc<int>;
+         streamp()->bzfree = Compression_Alloc_Info::free;
+         }
+
+      u32bit run_flag() const override { return BZ_RUN; }
+      u32bit flush_flag() const override { return BZ_FLUSH; }
+      u32bit finish_flag() const override { return BZ_FINISH; }
+   };
+
+class Bzip_Compression_Stream : public Bzip_Stream
+   {
+   public:
+      Bzip_Compression_Stream(size_t level)
+         {
+         int rc = BZ2_bzCompressInit(streamp(), level, 0, 0);
+
+         if(rc == BZ_MEM_ERROR)
+            throw std::bad_alloc();
+         else if(rc != BZ_OK)
+            throw std::runtime_error("bzip compress initialization failed");
+         }
+
+      ~Bzip_Compression_Stream()
+         {
+         BZ2_bzCompressEnd(streamp());
+         }
+
+      bool run(u32bit flags) override
+         {
+         int rc = BZ2_bzCompress(streamp(), flags);
+
+         if(rc == BZ_MEM_ERROR)
+            throw std::bad_alloc();
+         else if(rc < 0)
+            throw std::runtime_error("bzip compress error");
+
+         return (rc == BZ_STREAM_END);
+         }
+
+   private:
+      size_t m_level;
+   };
+
+class Bzip_Decompression_Stream : public Bzip_Stream
+   {
+   public:
+      Bzip_Decompression_Stream()
+         {
+         int rc = BZ2_bzDecompressInit(streamp(), 0, 0);
+
+         if(rc == BZ_MEM_ERROR)
+            throw std::bad_alloc();
+         else if(rc != BZ_OK)
+            throw std::runtime_error("bzip decompress initialization failed");
+         }
+
+      ~Bzip_Decompression_Stream()
+         {
+         BZ2_bzDecompressEnd(streamp());
+         }
+
+      bool run(u32bit flags) override
+         {
+         int rc = BZ2_bzDecompress(streamp());
+
+         if(rc == BZ_MEM_ERROR)
+            throw std::bad_alloc();
+         else if(rc != BZ_OK && rc != BZ_STREAM_END)
+            throw std::runtime_error("bzip decompress error");
+
+         return (rc == BZ_STREAM_END);
+         }
+   };
+
+}
+
+Compression_Stream* Bzip_Compression::make_stream() const
+   {
+   return new Bzip_Compression_Stream(m_level);
+   }
+
+Compression_Stream* Bzip_Decompression::make_stream() const
+   {
+   return new Bzip_Decompression_Stream;
+   }
+
+}
diff --git a/src/lib/compression/bzip2/bzip2.h b/src/lib/compression/bzip2/bzip2.h
new file mode 100644
index 000000000..945f6a051
--- /dev/null
+++ b/src/lib/compression/bzip2/bzip2.h
@@ -0,0 +1,50 @@
+/*
+* Bzip Compressor
+* (C) 2001 Peter J Jones
+*     2001-2007 Jack Lloyd
+*
+* Distributed under the terms of the Botan license
+*/
+
+#ifndef BOTAN_BZIP2_H__
+#define BOTAN_BZIP2_H__
+
+#include <botan/compression.h>
+
+namespace Botan {
+
+/**
+* Bzip Compression
+*/
+class BOTAN_DLL Bzip_Compression : public Stream_Compression
+   {
+   public:
+      /**
+      * @param level how much effort to use on compressing (0 to 9);
+      *        higher levels are slower but tend to give better
+      *        compression
+      */
+      Bzip_Compression(size_t level = 6) : m_level(level) {}
+
+      std::string name() const override { return "Bzip_Compression"; }
+
+   private:
+      Compression_Stream* make_stream() const;
+
+      const size_t m_level;
+   };
+
+/**
+* Bzip Deccompression
+*/
+class BOTAN_DLL Bzip_Decompression : public Stream_Decompression
+   {
+   public:
+      std::string name() const override { return "Bzip_Decompression"; }
+   private:
+      Compression_Stream* make_stream() const;
+   };
+
+}
+
+#endif
diff --git a/src/lib/compression/bzip2/info.txt b/src/lib/compression/bzip2/info.txt
new file mode 100644
index 000000000..a0f8d82ee
--- /dev/null
+++ b/src/lib/compression/bzip2/info.txt
@@ -0,0 +1,7 @@
+define BZIP2_TRANSFORM 20141118
+
+load_on request
+
+<libs>
+all -> bz2
+</libs>
diff --git a/src/lib/compression/comp_util.cpp b/src/lib/compression/comp_util.cpp
new file mode 100644
index 000000000..7fca1852d
--- /dev/null
+++ b/src/lib/compression/comp_util.cpp
@@ -0,0 +1,34 @@
+/*
+* Allocation Tracker
+* (C) 2014 Jack Lloyd
+*
+* Distributed under the terms of the Botan license
+*/
+
+#include <botan/internal/comp_util.h>
+#include <cstring>
+#include <cstdlib>
+
+namespace Botan {
+
+void* Compression_Alloc_Info::do_malloc(size_t n, size_t size)
+   {
+   const size_t total_sz = n * size;
+
+   void* ptr = std::malloc(total_sz);
+   m_current_allocs[ptr] = total_sz;
+   return ptr;
+   }
+
+void Compression_Alloc_Info::do_free(void* ptr)
+   {
+   auto i = m_current_allocs.find(ptr);
+   if(i == m_current_allocs.end())
+      throw std::runtime_error("Compression_Alloc_Info::free got pointer not allocated by us");
+
+   std::memset(ptr, 0, i->second);
+   std::free(ptr);
+   m_current_allocs.erase(i);
+   }
+
+}
diff --git a/src/lib/compression/comp_util.h b/src/lib/compression/comp_util.h
new file mode 100644
index 000000000..15fc23418
--- /dev/null
+++ b/src/lib/compression/comp_util.h
@@ -0,0 +1,90 @@
+/*
+* Shared code for compression libraries
+* (C) 2014 Jack Lloyd
+*
+* Distributed under the terms of the Botan license
+*/
+
+#ifndef BOTAN_COMPRESSION_UTILS_H__
+#define BOTAN_COMPRESSION_UTILS_H__
+
+#include <botan/compression.h>
+#include <cstring>
+#include <memory>
+#include <unordered_map>
+
+namespace Botan {
+
+/*
+* Allocation Size Tracking Helper for Zlib/Bzlib/LZMA
+*/
+class Compression_Alloc_Info
+   {
+   public:
+      template<typename T>
+      static void* malloc(void* self, T n, T size)
+         {
+         return static_cast<Compression_Alloc_Info*>(self)->do_malloc(n, size);
+         }
+
+      static void free(void* self, void* ptr)
+         {
+         static_cast<Compression_Alloc_Info*>(self)->do_free(ptr);
+         }
+
+   private:
+      void* do_malloc(size_t n, size_t size);
+      void do_free(void* ptr);
+
+      std::unordered_map<void*, size_t> m_current_allocs;
+   };
+
+/**
+* Wrapper for Zlib/Bzlib/LZMA stream types
+*/
+template<typename Stream, typename ByteType>
+class Zlib_Style_Stream : public Compression_Stream
+   {
+   public:
+      void next_in(byte* b, size_t len) override
+         {
+         m_stream.next_in = reinterpret_cast<ByteType*>(b);
+         m_stream.avail_in = len;
+         }
+
+      void next_out(byte* b, size_t len) override
+         {
+         m_stream.next_out = reinterpret_cast<ByteType*>(b);
+         m_stream.avail_out = len;
+         }
+
+      size_t avail_in() const override { return m_stream.avail_in; }
+
+      size_t avail_out() const override { return m_stream.avail_out; }
+
+      Zlib_Style_Stream()
+         {
+         std::memset(&m_stream, 0, sizeof(stream_t));
+         m_allocs.reset(new Compression_Alloc_Info);
+         }
+
+      ~Zlib_Style_Stream()
+         {
+         std::memset(&m_stream, 0, sizeof(stream_t));
+         m_allocs.reset();
+         }
+
+   protected:
+      typedef Stream stream_t;
+
+      stream_t* streamp() { return &m_stream; }
+
+      Compression_Alloc_Info* alloc() { return m_allocs.get(); }
+   private:
+      stream_t m_stream;
+      std::unique_ptr<Compression_Alloc_Info> m_allocs;
+   };
+
+}
+
+#endif
diff --git a/src/lib/compression/compression.cpp b/src/lib/compression/compression.cpp
new file mode 100644
index 000000000..3eb26e81b
--- /dev/null
+++ b/src/lib/compression/compression.cpp
@@ -0,0 +1,152 @@
+/*
+* Compression Transform
+* (C) 2014 Jack Lloyd
+*
+* Distributed under the terms of the Botan license
+*/
+
+#include <botan/compression.h>
+
+namespace Botan {
+
+void Stream_Compression::clear()
+   {
+   m_stream.reset();
+   }
+
+secure_vector<byte> Stream_Compression::start_raw(const byte[], size_t nonce_len)
+   {
+   if(!valid_nonce_length(nonce_len))
+      throw Invalid_IV_Length(name(), nonce_len);
+
+   clear();
+   m_stream.reset(make_stream());
+   return secure_vector<byte>();
+   }
+
+void Stream_Compression::process(secure_vector<byte>& buf, size_t offset, u32bit flags)
+   {
+   BOTAN_ASSERT(m_stream, "Initialized");
+   BOTAN_ASSERT(buf.size() >= offset, "Offset is sane");
+
+   if(m_buffer.size() < buf.size() + offset)
+      m_buffer.resize(buf.size() + offset);
+
+   m_stream->next_in(&buf[offset], buf.size() - offset);
+   m_stream->next_out(&m_buffer[offset], m_buffer.size() - offset);
+
+   while(true)
+      {
+      const bool end = m_stream->run(flags);
+
+      if(m_stream->avail_out() == 0)
+         {
+         const size_t added = 8 + m_buffer.size();
+         m_buffer.resize(m_buffer.size() + added);
+         m_stream->next_out(&m_buffer[m_buffer.size() - added], added);
+         }
+      else if(m_stream->avail_in() == 0)
+         {
+         m_buffer.resize(m_buffer.size() - m_stream->avail_out());
+         break;
+         }
+      }
+
+   copy_mem(&m_buffer[0], &buf[0], offset);
+   buf.swap(m_buffer);
+   }
+
+void Stream_Compression::update(secure_vector<byte>& buf, size_t offset)
+   {
+   process(buf, offset, m_stream->run_flag());
+   }
+
+void Stream_Compression::flush(secure_vector<byte>& buf, size_t offset)
+   {
+   process(buf, offset, m_stream->flush_flag());
+   }
+
+void Stream_Compression::finish(secure_vector<byte>& buf, size_t offset)
+   {
+   process(buf, offset, m_stream->finish_flag());
+   clear();
+   }
+
+void Stream_Decompression::clear()
+   {
+   m_stream.reset();
+   }
+
+secure_vector<byte> Stream_Decompression::start_raw(const byte[], size_t nonce_len)
+   {
+   if(!valid_nonce_length(nonce_len))
+      throw Invalid_IV_Length(name(), nonce_len);
+
+   clear();
+   m_stream.reset(make_stream());
+
+   return secure_vector<byte>();
+   }
+
+void Stream_Decompression::process(secure_vector<byte>& buf, size_t offset, u32bit flags)
+   {
+   BOTAN_ASSERT(m_stream, "Initialized");
+   BOTAN_ASSERT(buf.size() >= offset, "Offset is sane");
+
+   if(m_buffer.size() < buf.size() + offset)
+      m_buffer.resize(buf.size() + offset);
+
+   m_stream->next_in(&buf[offset], buf.size() - offset);
+   m_stream->next_out(&m_buffer[offset], m_buffer.size() - offset);
+
+   while(true)
+      {
+      const bool stream_end = m_stream->run(flags);
+
+      if(stream_end)
+         {
+         if(m_stream->avail_in() == 0) // all data consumed?
+            {
+            m_buffer.resize(m_buffer.size() - m_stream->avail_out());
+            clear();
+            break;
+            }
+
+         // More data follows: try to process as a following stream
+         const size_t read = (buf.size() - offset) - m_stream->avail_in();
+         start();
+         m_stream->next_in(&buf[offset + read], buf.size() - offset - read);
+         }
+
+      if(m_stream->avail_out() == 0)
+         {
+         const size_t added = 8 + m_buffer.size();
+         m_buffer.resize(m_buffer.size() + added);
+         m_stream->next_out(&m_buffer[m_buffer.size() - added], added);
+         }
+      else if(m_stream->avail_in() == 0)
+         {
+         m_buffer.resize(m_buffer.size() - m_stream->avail_out());
+         break;
+         }
+      }
+
+   copy_mem(&m_buffer[0], &buf[0], offset);
+   buf.swap(m_buffer);
+   }
+
+void Stream_Decompression::update(secure_vector<byte>& buf, size_t offset)
+   {
+   process(buf, offset, m_stream->run_flag());
+   }
+
+void Stream_Decompression::finish(secure_vector<byte>& buf, size_t offset)
+   {
+   if(buf.size() != offset || m_stream.get())
+      process(buf, offset, m_stream->finish_flag());
+
+   if(m_stream.get())
+      throw std::runtime_error(name() + " finished but not at stream end");
+   }
+
+}
diff --git a/src/lib/compression/compression.h b/src/lib/compression/compression.h
new file mode 100644
index 000000000..10c7f1a64
--- /dev/null
+++ b/src/lib/compression/compression.h
@@ -0,0 +1,99 @@
+/*
+* Compression Transform
+* (C) 2014 Jack Lloyd
+*
+* Distributed under the terms of the Botan license
+*/
+
+#ifndef BOTAN_COMPRESSION_TRANSFORM_H__
+#define BOTAN_COMPRESSION_TRANSFORM_H__
+
+#include <botan/transform.h>
+
+namespace Botan {
+
+class BOTAN_DLL Compressor_Transformation : public Transformation
+   {
+   public:
+      size_t update_granularity() const override { return 1; }
+
+      size_t minimum_final_size() const override { return 0; }
+
+      size_t default_nonce_length() const override { return 0; }
+
+      bool valid_nonce_length(size_t nonce_len) const override
+         { return nonce_len == 0; }
+
+      virtual void flush(secure_vector<byte>& buf, size_t offset = 0) { update(buf, offset); }
+
+      size_t output_length(size_t) const override
+         {
+         throw std::runtime_error(name() + " output length indeterminate");
+         }
+   };
+
+class Compression_Stream
+   {
+   public:
+      virtual ~Compression_Stream() {}
+
+      virtual void next_in(byte* b, size_t len) = 0;
+
+      virtual void next_out(byte* b, size_t len) = 0;
+
+      virtual size_t avail_in() const = 0;
+
+      virtual size_t avail_out() const = 0;
+
+      virtual u32bit run_flag() const = 0;
+      virtual u32bit flush_flag() const = 0;
+      virtual u32bit finish_flag() const = 0;
+
+      virtual bool run(u32bit flags) = 0;
+   };
+
+class BOTAN_DLL Stream_Compression : public Compressor_Transformation
+   {
+   public:
+      void update(secure_vector<byte>& buf, size_t offset = 0) override;
+
+      void flush(secure_vector<byte>& buf, size_t offset = 0) override;
+
+      void finish(secure_vector<byte>& buf, size_t offset = 0) override;
+
+      void clear() override;
+
+   private:
+      secure_vector<byte> start_raw(const byte[], size_t) override;
+
+      void process(secure_vector<byte>& buf, size_t offset, u32bit flags);
+
+      virtual Compression_Stream* make_stream() const = 0;
+
+      secure_vector<byte> m_buffer;
+      std::unique_ptr<Compression_Stream> m_stream;
+   };
+
+class BOTAN_DLL Stream_Decompression : public Compressor_Transformation
+   {
+   public:
+      void update(secure_vector<byte>& buf, size_t offset = 0) override;
+
+      void finish(secure_vector<byte>& buf, size_t offset = 0) override;
+
+      void clear() override;
+
+   private:
+      secure_vector<byte> start_raw(const byte[], size_t) override;
+
+      void process(secure_vector<byte>& buf, size_t offset, u32bit flags);
+
+      virtual Compression_Stream* make_stream() const = 0;
+
+      secure_vector<byte> m_buffer;
+      std::unique_ptr<Compression_Stream> m_stream;
+   };
+
+}
+
+#endif
diff --git a/src/lib/compression/info.txt b/src/lib/compression/info.txt
new file mode 100644
index 000000000..f1a3fa696
--- /dev/null
+++ b/src/lib/compression/info.txt
@@ -0,0 +1,9 @@
+define COMPRESSION 20141117
+
+<header:internal>
+comp_util.h
+</header:internal>
+
+<header:public>
+compression.h
+</header:public>
diff --git a/src/lib/compression/lzma/info.txt b/src/lib/compression/lzma/info.txt
new file mode 100644
index 000000000..1ec668ca8
--- /dev/null
+++ b/src/lib/compression/lzma/info.txt
@@ -0,0 +1,7 @@
+define LZMA_TRANSFORM 20141118
+
+load_on request
+
+<libs>
+all -> lzma
+</libs>
diff --git a/src/lib/compression/lzma/lzma.cpp b/src/lib/compression/lzma/lzma.cpp
new file mode 100644
index 000000000..d145ed305
--- /dev/null
+++ b/src/lib/compression/lzma/lzma.cpp
@@ -0,0 +1,94 @@
+/*
+* Lzma Compressor
+* (C) 2001 Peter J Jones
+*     2001-2007,2014 Jack Lloyd
+*     2006 Matt Johnston
+*     2012 Vojtech Kral
+*
+* Distributed under the terms of the Botan license
+*/
+
+#include <botan/lzma.h>
+#include <botan/internal/comp_util.h>
+#include <lzma.h>
+
+namespace Botan {
+
+namespace {
+
+class LZMA_Stream : public Zlib_Style_Stream<lzma_stream, byte>
+   {
+   public:
+      LZMA_Stream()
+         {
+         streamp()->allocator = new ::lzma_allocator;
+         streamp()->allocator->opaque = alloc();
+         streamp()->allocator->alloc = Compression_Alloc_Info::malloc<size_t>;
+         streamp()->allocator->free = Compression_Alloc_Info::free;
+         }
+
+      ~LZMA_Stream()
+         {
+         ::lzma_end(streamp());
+         delete streamp()->allocator;
+         }
+
+      bool run(u32bit flags) override
+         {
+         lzma_ret rc = ::lzma_code(streamp(), static_cast<lzma_action>(flags));
+
+         if(rc == LZMA_MEM_ERROR)
+            throw std::bad_alloc();
+         else if (rc != LZMA_OK && rc != LZMA_STREAM_END)
+            throw std::runtime_error("Lzma error");
+
+         return (rc == LZMA_STREAM_END);
+         }
+
+      u32bit run_flag() const override { return LZMA_RUN; }
+      u32bit flush_flag() const override { return LZMA_FULL_FLUSH; }
+      u32bit finish_flag() const override { return LZMA_FINISH; }
+   };
+
+class LZMA_Compression_Stream : public LZMA_Stream
+   {
+   public:
+      LZMA_Compression_Stream(size_t level)
+         {
+         lzma_ret rc = ::lzma_easy_encoder(streamp(), level, LZMA_CHECK_CRC64);
+
+         if(rc == LZMA_MEM_ERROR)
+            throw std::bad_alloc();
+         else if(rc != LZMA_OK)
+            throw std::runtime_error("lzma compress initialization failed");
+         }
+   };
+
+class LZMA_Decompression_Stream : public LZMA_Stream
+   {
+   public:
+      LZMA_Decompression_Stream()
+         {
+         lzma_ret rc = ::lzma_stream_decoder(streamp(), UINT64_MAX,
+                                             LZMA_TELL_UNSUPPORTED_CHECK);
+
+         if(rc == LZMA_MEM_ERROR)
+            throw std::bad_alloc();
+         else if(rc != LZMA_OK)
+            throw std::runtime_error("Bad setting in lzma_stream_decoder");
+         }
+   };
+
+}
+
+Compression_Stream* LZMA_Compression::make_stream() const
+   {
+   return new LZMA_Compression_Stream(m_level);
+   }
+
+Compression_Stream* LZMA_Decompression::make_stream() const
+   {
+   return new LZMA_Decompression_Stream;
+   }
+
+}
diff --git a/src/lib/compression/lzma/lzma.h b/src/lib/compression/lzma/lzma.h
new file mode 100644
index 000000000..0761a8194
--- /dev/null
+++ b/src/lib/compression/lzma/lzma.h
@@ -0,0 +1,51 @@
+/*
+* Lzma Compressor
+* (C) 2001 Peter J Jones
+*     2001-2007 Jack Lloyd
+*     2012 Vojtech Kral
+*
+* Distributed under the terms of the Botan license
+*/
+
+#ifndef BOTAN_LZMA_H__
+#define BOTAN_LZMA_H__
+
+#include <botan/compression.h>
+
+namespace Botan {
+
+/**
+* LZMA Compression
+*/
+class BOTAN_DLL LZMA_Compression : public Stream_Compression
+   {
+   public:
+      /**
+      * @param level how much effort to use on compressing (0 to 9);
+      *        higher levels are slower but tend to give better
+      *        compression
+      */
+      LZMA_Compression(size_t level = 6) : m_level(level) {}
+
+      std::string name() const override { return "LZMA_Compression"; }
+
+   private:
+      Compression_Stream* make_stream() const;
+
+      const size_t m_level;
+   };
+
+/**
+* LZMA Deccompression
+*/
+class BOTAN_DLL LZMA_Decompression : public Stream_Decompression
+   {
+   public:
+      std::string name() const override { return "LZMA_Decompression"; }
+   private:
+      Compression_Stream* make_stream() const;
+   };
+
+}
+
+#endif
diff --git a/src/lib/compression/zlib/info.txt b/src/lib/compression/zlib/info.txt
new file mode 100644
index 000000000..ba5abd4ec
--- /dev/null
+++ b/src/lib/compression/zlib/info.txt
@@ -0,0 +1,7 @@
+define ZLIB_TRANSFORM 20141118
+
+load_on request
+
+<libs>
+all -> z
+</libs>
diff --git a/src/lib/compression/zlib/zlib.cpp b/src/lib/compression/zlib/zlib.cpp
new file mode 100644
index 000000000..b18621640
--- /dev/null
+++ b/src/lib/compression/zlib/zlib.cpp
@@ -0,0 +1,106 @@
+/*
+* Zlib Compressor
+* (C) 2001 Peter J Jones
+*     2001-2007,2014 Jack Lloyd
+*     2006 Matt Johnston
+*
+* Distributed under the terms of the Botan license
+*/
+
+#include <botan/zlib.h>
+#include <botan/internal/comp_util.h>
+#include <zlib.h>
+
+namespace Botan {
+
+namespace {
+
+class Zlib_Stream : public Zlib_Style_Stream<z_stream, Bytef>
+   {
+   public:
+      Zlib_Stream()
+         {
+         streamp()->opaque = alloc();
+         streamp()->zalloc = Compression_Alloc_Info::malloc<unsigned int>;
+         streamp()->zfree = Compression_Alloc_Info::free;
+         }
+
+      u32bit run_flag() const override { return Z_NO_FLUSH; }
+      u32bit flush_flag() const override { return Z_FULL_FLUSH; }
+      u32bit finish_flag() const override { return Z_FINISH; }
+   };
+
+class Zlib_Compression_Stream : public Zlib_Stream
+   {
+   public:
+      Zlib_Compression_Stream(size_t level, bool raw_deflate)
+         {
+         // FIXME: allow specifiying memLevel and strategy
+         int rc = deflateInit2(streamp(), level, Z_DEFLATED,
+                               (raw_deflate ? -15 : 15), 8, Z_DEFAULT_STRATEGY);
+         if(rc != Z_OK)
+            throw std::runtime_error("zlib deflate initialization failed");
+         }
+
+      ~Zlib_Compression_Stream()
+         {
+         deflateEnd(streamp());
+         }
+
+      bool run(u32bit flags) override
+         {
+         int rc = deflate(streamp(), flags);
+
+         if(rc == Z_MEM_ERROR)
+            throw std::bad_alloc();
+         else if(rc != Z_OK && rc != Z_STREAM_END)
+            throw std::runtime_error("zlib deflate error");
+
+         return (rc == Z_STREAM_END);
+         }
+   };
+
+class Zlib_Decompression_Stream : public Zlib_Stream
+   {
+   public:
+      Zlib_Decompression_Stream(bool raw_deflate)
+         {
+         int rc = inflateInit2(streamp(), (raw_deflate ? -15 : 15));
+
+         if(rc == Z_MEM_ERROR)
+            throw std::bad_alloc();
+         else if(rc != Z_OK)
+            throw std::runtime_error("zlib inflate initialization failed");
+         }
+
+      ~Zlib_Decompression_Stream()
+         {
+         inflateEnd(streamp());
+         }
+
+      bool run(u32bit flags) override
+         {
+         int rc = inflate(streamp(), flags);
+
+         if(rc == Z_MEM_ERROR)
+            throw std::bad_alloc();
+         else if(rc != Z_OK && rc != Z_STREAM_END)
+            throw std::runtime_error("zlib deflate error");
+
+         return (rc == Z_STREAM_END);
+         }
+   };
+
+}
+
+Compression_Stream* Zlib_Compression::make_stream() const
+   {
+   return new Zlib_Compression_Stream(m_level, m_raw_deflate);
+   }
+
+Compression_Stream* Zlib_Decompression::make_stream() const
+   {
+   return new Zlib_Decompression_Stream(m_raw_deflate);
+   }
+
+}
diff --git a/src/lib/compression/zlib/zlib.h b/src/lib/compression/zlib/zlib.h
new file mode 100644
index 000000000..55da47a0d
--- /dev/null
+++ b/src/lib/compression/zlib/zlib.h
@@ -0,0 +1,61 @@
+/*
+* Zlib Compressor
+* (C) 2001 Peter J Jones
+*     2001-2007 Jack Lloyd
+*
+* Distributed under the terms of the Botan license
+*/
+
+#ifndef BOTAN_ZLIB_H__
+#define BOTAN_ZLIB_H__
+
+#include <botan/compression.h>
+
+namespace Botan {
+
+/**
+* Zlib Compression
+*/
+class BOTAN_DLL Zlib_Compression : public Stream_Compression
+   {
+   public:
+      /**
+      * @param level how much effort to use on compressing (0 to 9);
+      *        higher levels are slower but tend to give better
+      *        compression
+      * @param raw_deflate if true no zlib header/trailer will be used
+      */
+      Zlib_Compression(size_t level = 6, bool raw_deflate = false) :
+         m_level(level), m_raw_deflate(raw_deflate) {}
+
+      std::string name() const override { return "Zlib_Compression"; }
+
+   private:
+      Compression_Stream* make_stream() const;
+
+      const size_t m_level;
+      const bool m_raw_deflate;
+   };
+
+/**
+* Zlib Deccompression
+*/
+class BOTAN_DLL Zlib_Decompression : public Stream_Decompression
+   {
+   public:
+      /**
+      * @param raw_deflate if true no zlib header/trailer will be used
+      */
+      Zlib_Decompression( bool raw_deflate = false) : m_raw_deflate(raw_deflate) {}
+
+      std::string name() const override { return "Zlib_Decompression"; }
+
+   private:
+      Compression_Stream* make_stream() const;
+
+      const bool m_raw_deflate;
+   };
+
+}
+
+#endif
diff --git a/src/lib/filters/comp_filter.cpp b/src/lib/filters/comp_filter.cpp
new file mode 100644
index 000000000..9c27978bc
--- /dev/null
+++ b/src/lib/filters/comp_filter.cpp
@@ -0,0 +1,127 @@
+/*
+* Filter interface for compression
+* (C) 2014 Jack Lloyd
+*
+* Distributed under the terms of the Botan license
+*/
+
+#include <botan/comp_filter.h>
+
+#if defined(BOTAN_HAS_ZLIB_TRANSFORM)
+  #include <botan/zlib.h>
+#endif
+
+#if defined(BOTAN_HAS_BZIP_TRANSFORM)
+  #include <botan/bzip.h>
+#endif
+
+#if defined(BOTAN_HAS_LZMA_TRANSFORM)
+  #include <botan/lzma.h>
+#endif
+
+namespace Botan {
+
+namespace {
+
+Compressor_Transformation* make_compressor(const std::string& type, size_t level)
+   {
+#if defined(BOTAN_HAS_ZLIB_TRANSFORM)
+   if(type == "zlib")
+      return new Zlib_Compression(level, false);
+   if(type == "deflate")
+      return new Zlib_Compression(level, true);
+#endif
+
+#if defined(BOTAN_HAS_BZIP_TRANSFORM)
+   if(type == "bzip2")
+      return new Bzip_Compression(level);
+#endif
+
+#if defined(BOTAN_HAS_LZMA_TRANSFORM)
+   if(type == "lzma")
+      return new LZMA_Compression(level);
+#endif
+
+   throw std::runtime_error("Unknown compression type " + type);
+   }
+
+Compressor_Transformation* make_decompressor(const std::string& type)
+   {
+#if defined(BOTAN_HAS_ZLIB_TRANSFORM)
+   if(type == "zlib")
+      return new Zlib_Decompression(false);
+   if(type == "deflate")
+      return new Zlib_Decompression(true);
+#endif
+
+#if defined(BOTAN_HAS_BZIP_TRANSFORM)
+   if(type == "bzip2")
+      return new Bzip_Decompression;
+#endif
+
+#if defined(BOTAN_HAS_LZMA_TRANSFORM)
+   if(type == "lzma")
+      return new LZMA_Decompression;
+#endif
+
+   throw std::runtime_error("Unknown compression type " + type);
+   }
+
+}
+
+Compression_Filter::Compression_Filter(const std::string& type, size_t level) :
+   Compression_Decompression_Filter(make_compressor(type, level))
+   {
+   }
+
+Decompression_Filter::Decompression_Filter(const std::string& type) :
+   Compression_Decompression_Filter(make_decompressor(type))
+   {
+   }
+
+Compression_Decompression_Filter::Compression_Decompression_Filter(Compressor_Transformation* transform) :
+   m_transform(transform)
+   {
+   }
+
+std::string Compression_Decompression_Filter::name() const
+   {
+   return m_transform->name();
+   }
+
+void Compression_Decompression_Filter::start_msg()
+   {
+   send(m_transform->start());
+   }
+
+void Compression_Decompression_Filter::write(const byte input[], size_t input_length)
+   {
+   while(input_length)
+      {
+      const size_t take = std::min<size_t>({4096, m_buffer.capacity(), input_length});
+
+      m_buffer.assign(input, input + take);
+      m_transform->update(m_buffer);
+
+      send(m_buffer);
+
+      input += take;
+      input_length -= take;
+      }
+   }
+
+void Compression_Decompression_Filter::flush()
+   {
+   m_buffer.clear();
+   m_transform->flush(m_buffer);
+   send(m_buffer);
+   }
+
+void Compression_Decompression_Filter::end_msg()
+   {
+   m_buffer.clear();
+   m_transform->finish(m_buffer);
+   send(m_buffer);
+   }
+
+}
diff --git a/src/lib/filters/comp_filter.h b/src/lib/filters/comp_filter.h
new file mode 100644
index 000000000..a46932bed
--- /dev/null
+++ b/src/lib/filters/comp_filter.h
@@ -0,0 +1,53 @@
+/*
+* Filter interface for compression
+* (C) 2014 Jack Lloyd
+*
+* Distributed under the terms of the Botan license
+*/
+
+#ifndef BOTAN_COMPRESSION_FILTER_H__
+#define BOTAN_COMPRESSION_FILTER_H__
+
+#include <botan/filter.h>
+#include <botan/compression.h>
+
+namespace Botan {
+
+/**
+* Filter interface for compression/decompression
+*/
+class BOTAN_DLL Compression_Decompression_Filter : public Filter
+   {
+   public:
+      void start_msg() override;
+      void write(const byte input[], size_t input_length) override;
+      void end_msg() override;
+
+      std::string name() const override;
+
+   protected:
+      Compression_Decompression_Filter(Compressor_Transformation* t);
+
+      void flush();
+   private:
+      std::unique_ptr<Compressor_Transformation> m_transform;
+      secure_vector<byte> m_buffer;
+   };
+
+class BOTAN_DLL Compression_Filter : public Compression_Decompression_Filter
+   {
+   public:
+      Compression_Filter(const std::string& type, size_t level); // compression
+
+      using Compression_Decompression_Filter::flush;
+   };
+
+class Decompression_Filter : public Compression_Decompression_Filter
+   {
+   public:
+      Decompression_Filter(const std::string& type);
+   };
+
+}
+
+#endif
diff --git a/src/lib/filters/compression/bzip2/bzip2.cpp b/src/lib/filters/compression/bzip2/bzip2.cpp
deleted file mode 100644
index a7990d2c2..000000000
--- a/src/lib/filters/compression/bzip2/bzip2.cpp
+++ /dev/null
@@ -1,298 +0,0 @@
-/*
-* Bzip Compressor
-* (C) 2001 Peter J Jones
-*     2001-2007 Jack Lloyd
-*     2006 Matt Johnston
-*
-* Distributed under the terms of the Botan license
-*/
-
-#include <botan/bzip2.h>
-#include <botan/exceptn.h>
-
-#include <map>
-#include <cstring>
-#include <cstdlib>
-#define BZ_NO_STDIO
-#include <bzlib.h>
-
-namespace Botan {
-
-namespace {
-
-/*
-* Allocation Information for Bzip
-*/
-class Bzip_Alloc_Info
-   {
-   public:
-      std::map<void*, size_t> current_allocs;
-   };
-
-/*
-* Allocation Function for Bzip
-*/
-void* bzip_malloc(void* info_ptr, int n, int size)
-   {
-   Bzip_Alloc_Info* info = static_cast<Bzip_Alloc_Info*>(info_ptr);
-
-   const size_t total_sz = n * size;
-
-   void* ptr = std::malloc(total_sz);
-   info->current_allocs[ptr] = total_sz;
-   return ptr;
-   }
-
-/*
-* Allocation Function for Bzip
-*/
-void bzip_free(void* info_ptr, void* ptr)
-   {
-   Bzip_Alloc_Info* info = static_cast<Bzip_Alloc_Info*>(info_ptr);
-   auto i = info->current_allocs.find(ptr);
-   if(i == info->current_allocs.end())
-      throw Invalid_Argument("bzip_free: Got pointer not allocated by us");
-
-   std::memset(ptr, 0, i->second);
-   std::free(ptr);
-   }
-
-}
-
-/**
-* Wrapper Type for Bzip2 Stream
-*/
-class Bzip_Stream
-   {
-   public:
-      /**
-      * Underlying stream
-      */
-      bz_stream stream;
-
-      /**
-      * Constructor
-      */
-      Bzip_Stream()
-         {
-         std::memset(&stream, 0, sizeof(bz_stream));
-         stream.bzalloc = bzip_malloc;
-         stream.bzfree = bzip_free;
-         stream.opaque = new Bzip_Alloc_Info;
-         }
-
-      /**
-      * Destructor
-      */
-      ~Bzip_Stream()
-         {
-         Bzip_Alloc_Info* info = static_cast<Bzip_Alloc_Info*>(stream.opaque);
-         delete info;
-         std::memset(&stream, 0, sizeof(bz_stream));
-         }
-   };
-
-/*
-* Bzip_Compression Constructor
-*/
-Bzip_Compression::Bzip_Compression(size_t l) :
-   level((l >= 9) ? 9 : l), buffer(DEFAULT_BUFFERSIZE)
-   {
-   bz = 0;
-   }
-
-/*
-* Start Compressing with Bzip
-*/
-void Bzip_Compression::start_msg()
-   {
-   clear();
-   bz = new Bzip_Stream;
-   if(BZ2_bzCompressInit(&(bz->stream), level, 0, 0) != BZ_OK)
-      throw Memory_Exhaustion();
-   }
-
-/*
-* Compress Input with Bzip
-*/
-void Bzip_Compression::write(const byte input[], size_t length)
-   {
-   bz->stream.next_in = reinterpret_cast<char*>(const_cast<byte*>(input));
-   bz->stream.avail_in = length;
-
-   while(bz->stream.avail_in != 0)
-      {
-      bz->stream.next_out = reinterpret_cast<char*>(&buffer[0]);
-      bz->stream.avail_out = buffer.size();
-      BZ2_bzCompress(&(bz->stream), BZ_RUN);
-      send(buffer, buffer.size() - bz->stream.avail_out);
-      }
-   }
-
-/*
-* Finish Compressing with Bzip
-*/
-void Bzip_Compression::end_msg()
-   {
-   bz->stream.next_in = 0;
-   bz->stream.avail_in = 0;
-
-   int rc = BZ_OK;
-   while(rc != BZ_STREAM_END)
-      {
-      bz->stream.next_out = reinterpret_cast<char*>(&buffer[0]);
-      bz->stream.avail_out = buffer.size();
-      rc = BZ2_bzCompress(&(bz->stream), BZ_FINISH);
-      send(buffer, buffer.size() - bz->stream.avail_out);
-      }
-   clear();
-   }
-
-/*
-* Flush the Bzip Compressor
-*/
-void Bzip_Compression::flush()
-   {
-   bz->stream.next_in = 0;
-   bz->stream.avail_in = 0;
-
-   int rc = BZ_OK;
-   while(rc != BZ_RUN_OK)
-      {
-      bz->stream.next_out = reinterpret_cast<char*>(&buffer[0]);
-      bz->stream.avail_out = buffer.size();
-      rc = BZ2_bzCompress(&(bz->stream), BZ_FLUSH);
-      send(buffer, buffer.size() - bz->stream.avail_out);
-      }
-   }
-
-/*
-* Clean up Compression Context
-*/
-void Bzip_Compression::clear()
-   {
-   zeroise(buffer);
-
-   if(bz)
-      {
-      BZ2_bzCompressEnd(&(bz->stream));
-      delete bz;
-      bz = 0;
-      }
-   }
-
-/*
-* Bzip_Decompression Constructor
-*/
-Bzip_Decompression::Bzip_Decompression(bool s) :
-   small_mem(s), buffer(DEFAULT_BUFFERSIZE)
-   {
-   no_writes = true;
-   bz = 0;
-   }
-
-/*
-* Decompress Input with Bzip
-*/
-void Bzip_Decompression::write(const byte input_arr[], size_t length)
-   {
-   if(length) no_writes = false;
-
-   char* input = reinterpret_cast<char*>(const_cast<byte*>(input_arr));
-
-   bz->stream.next_in = input;
-   bz->stream.avail_in = length;
-
-   while(bz->stream.avail_in != 0)
-      {
-      bz->stream.next_out = reinterpret_cast<char*>(&buffer[0]);
-      bz->stream.avail_out = buffer.size();
-
-      int rc = BZ2_bzDecompress(&(bz->stream));
-
-      if(rc != BZ_OK && rc != BZ_STREAM_END)
-         {
-         clear();
-
-         if(rc == BZ_DATA_ERROR)
-            throw Decoding_Error("Bzip_Decompression: Data integrity error");
-         else if(rc == BZ_DATA_ERROR_MAGIC)
-            throw Decoding_Error("Bzip_Decompression: Invalid input");
-         else if(rc == BZ_MEM_ERROR)
-            throw Memory_Exhaustion();
-         else
-            throw std::runtime_error("Bzip2 decompression: Unknown error");
-         }
-
-      send(buffer, buffer.size() - bz->stream.avail_out);
-
-      if(rc == BZ_STREAM_END)
-         {
-         size_t read_from_block = length - bz->stream.avail_in;
-         start_msg();
-         bz->stream.next_in = input + read_from_block;
-         bz->stream.avail_in = length - read_from_block;
-         input += read_from_block;
-         length -= read_from_block;
-         }
-      }
-   }
-
-/*
-* Start Decompressing with Bzip
-*/
-void Bzip_Decompression::start_msg()
-   {
-   clear();
-   bz = new Bzip_Stream;
-
-   if(BZ2_bzDecompressInit(&(bz->stream), 0, small_mem) != BZ_OK)
-      throw Memory_Exhaustion();
-
-   no_writes = true;
-   }
-
-/*
-* Finish Decompressing with Bzip
-*/
-void Bzip_Decompression::end_msg()
-   {
-   if(no_writes) return;
-   bz->stream.next_in = 0;
-   bz->stream.avail_in = 0;
-
-   int rc = BZ_OK;
-   while(rc != BZ_STREAM_END)
-      {
-      bz->stream.next_out = reinterpret_cast<char*>(&buffer[0]);
-      bz->stream.avail_out = buffer.size();
-      rc = BZ2_bzDecompress(&(bz->stream));
-
-      if(rc != BZ_OK && rc != BZ_STREAM_END)
-         {
-         clear();
-         throw Decoding_Error("Bzip_Decompression: Error finalizing");
-         }
-
-      send(buffer, buffer.size() - bz->stream.avail_out);
-      }
-
-   clear();
-   }
-
-/*
-* Clean up Decompression Context
-*/
-void Bzip_Decompression::clear()
-   {
-   zeroise(buffer);
-
-   if(bz)
-      {
-      BZ2_bzDecompressEnd(&(bz->stream));
-      delete bz;
-      bz = 0;
-      }
-   }
-
-}
diff --git a/src/lib/filters/compression/bzip2/bzip2.h b/src/lib/filters/compression/bzip2/bzip2.h
deleted file mode 100644
index 2505cf54e..000000000
--- a/src/lib/filters/compression/bzip2/bzip2.h
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
-* Bzip Compressor
-* (C) 2001 Peter J Jones
-*     2001-2007 Jack Lloyd
-*
-* Distributed under the terms of the Botan license
-*/
-
-#ifndef BOTAN_BZIP2_H__
-#define BOTAN_BZIP2_H__
-
-#include <botan/filter.h>
-
-namespace Botan {
-
-/**
-* Bzip Compression Filter
-*/
-class BOTAN_DLL Bzip_Compression : public Filter
-   {
-   public:
-      std::string name() const { return "Bzip_Compression"; }
-
-      void write(const byte input[], size_t length);
-      void start_msg();
-      void end_msg();
-
-      void flush();
-
-      Bzip_Compression(size_t = 9);
-      ~Bzip_Compression() { clear(); }
-   private:
-      void clear();
-
-      const size_t level;
-      secure_vector<byte> buffer;
-      class Bzip_Stream* bz;
-   };
-
-/**
-* Bzip Decompression Filter
-*/
-class BOTAN_DLL Bzip_Decompression : public Filter
-   {
-   public:
-      std::string name() const { return "Bzip_Decompression"; }
-
-      void write(const byte input[], size_t length);
-      void start_msg();
-      void end_msg();
-
-      Bzip_Decompression(bool = false);
-      ~Bzip_Decompression() { clear(); }
-   private:
-      void clear();
-
-      const bool small_mem;
-      secure_vector<byte> buffer;
-      class Bzip_Stream* bz;
-      bool no_writes;
-   };
-
-}
-
-#endif
diff --git a/src/lib/filters/compression/bzip2/info.txt b/src/lib/filters/compression/bzip2/info.txt
deleted file mode 100644
index 7c0f2aafc..000000000
--- a/src/lib/filters/compression/bzip2/info.txt
+++ /dev/null
@@ -1,11 +0,0 @@
-define COMPRESSOR_BZIP2 20131128
-
-load_on request
-
-<libs>
-all -> bz2
-</libs>
-
-<requires>
-filters
-</requires>
diff --git a/src/lib/filters/compression/lzma/info.txt b/src/lib/filters/compression/lzma/info.txt
deleted file mode 100644
index ccd7a7600..000000000
--- a/src/lib/filters/compression/lzma/info.txt
+++ /dev/null
@@ -1,11 +0,0 @@
-define COMPRESSOR_LZMA 20131128
-
-load_on request
-
-<libs>
-all -> lzma
-</libs>
-
-<requires>
-filters
-</requires>
diff --git a/src/lib/filters/compression/lzma/lzma.cpp b/src/lib/filters/compression/lzma/lzma.cpp
deleted file mode 100644
index eaf18b4bf..000000000
--- a/src/lib/filters/compression/lzma/lzma.cpp
+++ /dev/null
@@ -1,327 +0,0 @@
-/*
-* Lzma Compressor
-* (C) 2001 Peter J Jones
-*     2001-2007 Jack Lloyd
-*     2006 Matt Johnston
-*     2012 Vojtech Kral
-*
-* Distributed under the terms of the Botan license
-*/
-
-#include <botan/lzma.h>
-#include <botan/exceptn.h>
-
-#include <cstring>
-#include <cstdlib>
-#include <map>
-#include <lzma.h>
-
-namespace Botan {
-
-namespace {
-
-/*
-* Allocation Information for Lzma
-*/
-class Lzma_Alloc_Info
-   {
-   public:
-      std::map<void*, size_t> current_allocs;
-   };
-
-/*
-* Allocation Function for Lzma
-*/
-void* lzma_malloc(void *opaque, size_t /*nmemb*/, size_t size)
-   {
-   Lzma_Alloc_Info* info = static_cast<Lzma_Alloc_Info*>(opaque);
-   void* ptr = std::malloc(size);      // It is guaranteed by liblzma doc that nmemb is always set to 1
-   info->current_allocs[ptr] = size;
-   return ptr;
-   }
-
-/*
-* Allocation Function for Lzma
-*/
-void lzma_free(void *opaque, void *ptr)
-   {
-   if(!ptr) return;       // liblzma sometimes does pass zero ptr
-
-   Lzma_Alloc_Info* info = static_cast<Lzma_Alloc_Info*>(opaque);
-   auto i = info->current_allocs.find(ptr);
-   if(i == info->current_allocs.end())
-      throw Invalid_Argument("lzma_free: Got pointer not allocated by us");
-
-   std::memset(ptr, 0, i->second);
-   std::free(ptr);
-   }
-
-}
-
-/**
-* Wrapper Type for lzma_stream
-*/
-class Lzma_Stream
-   {
-   public:
-      /**
-      * Underlying stream
-      */
-      lzma_stream stream;
-
-      /**
-      * Constructor
-      */
-      Lzma_Stream() :
-         stream(LZMA_STREAM_INIT)
-         {
-         stream.allocator = new lzma_allocator;
-         stream.allocator->alloc = lzma_malloc;
-         stream.allocator->free = lzma_free;
-         stream.allocator->opaque = new Lzma_Alloc_Info;
-         }
-
-      /**
-      * Destructor
-      */
-      ~Lzma_Stream()
-         {
-         Lzma_Alloc_Info* info = static_cast<Lzma_Alloc_Info*>(stream.allocator->opaque);
-         delete info;
-         delete stream.allocator;
-         std::memset(&stream, 0, sizeof(lzma_stream));
-         }
-   };
-
-/*
-* Lzma_Compression Constructor
-*/
-Lzma_Compression::Lzma_Compression(size_t l) :
-   level((l >= 9) ? 9 : l),
-   buffer(DEFAULT_BUFFERSIZE),
-   lzma(0)
-   {
-   }
-
-/*
-* Start Compressing with Lzma
-*/
-void Lzma_Compression::start_msg()
-   {
-   clear();
-   lzma = new Lzma_Stream;
-
-   lzma_ret ret = lzma_easy_encoder(&(lzma->stream), level, LZMA_CHECK_CRC64);
-
-   if(ret == LZMA_MEM_ERROR)
-      throw Memory_Exhaustion();
-   else if(ret != LZMA_OK)
-      throw Invalid_Argument("Bad setting in lzma_easy_encoder");
-   }
-
-/*
-* Compress Input with Lzma
-*/
-void Lzma_Compression::write(const byte input[], size_t length)
-   {
-   lzma->stream.next_in = static_cast<const uint8_t*>(input);
-   lzma->stream.avail_in = length;
-
-   while(lzma->stream.avail_in != 0)
-      {
-      lzma->stream.next_out = static_cast<uint8_t*>(&buffer[0]);
-      lzma->stream.avail_out = buffer.size();
-
-      lzma_ret ret = lzma_code(&(lzma->stream), LZMA_RUN);
-
-      if(ret == LZMA_MEM_ERROR)
-         throw Memory_Exhaustion();
-      else if (ret != LZMA_OK)
-         throw std::runtime_error("Lzma compression: Error writing");
-
-      send(&buffer[0], buffer.size() - lzma->stream.avail_out);
-      }
-   }
-
-/*
-* Finish Compressing with Lzma
-*/
-void Lzma_Compression::end_msg()
-   {
-   lzma->stream.next_in = 0;
-   lzma->stream.avail_in = 0;
-
-   int ret = LZMA_OK;
-   while(ret != LZMA_STREAM_END)
-      {
-      lzma->stream.next_out = reinterpret_cast<uint8_t*>(&buffer[0]);
-      lzma->stream.avail_out = buffer.size();
-
-      ret = lzma_code(&(lzma->stream), LZMA_FINISH);
-      send(&buffer[0], buffer.size() - lzma->stream.avail_out);
-      }
-
-   clear();
-   }
-
-/*
-* Flush the Lzma Compressor
-*/
-void Lzma_Compression::flush()
-   {
-   lzma->stream.next_in = 0;
-   lzma->stream.avail_in = 0;
-
-   while(true)
-      {
-      lzma->stream.next_out = reinterpret_cast<uint8_t*>(&buffer[0]);
-      lzma->stream.avail_out = buffer.size();
-
-      lzma_ret ret = lzma_code(&(lzma->stream), LZMA_FULL_FLUSH);
-
-      if(ret == LZMA_MEM_ERROR)
-         throw Memory_Exhaustion();
-      else if (ret != LZMA_OK && ret != LZMA_STREAM_END)
-         throw std::runtime_error("Lzma compression: Error flushing");
-
-      send(&buffer[0], buffer.size() - lzma->stream.avail_out);
-
-      if(lzma->stream.avail_out == buffer.size())
-         break;
-      }
-   }
-
-/*
-* Clean up Compression Context
-*/
-void Lzma_Compression::clear()
-   {
-   zeroise(buffer);
-
-   if(lzma)
-      {
-      lzma_end(&(lzma->stream));
-      delete lzma;
-      lzma = 0;
-      }
-   }
-
-/*
-* Lzma_Decompression Constructor
-*/
-Lzma_Decompression::Lzma_Decompression() :
-   buffer(DEFAULT_BUFFERSIZE),
-   lzma(0),
-   no_writes(true)
-   {
-   }
-
-/*
-* Start Decompressing with Lzma
-*/
-void Lzma_Decompression::start_msg()
-   {
-   clear();
-   lzma = new Lzma_Stream;
-
-   lzma_ret ret = lzma_stream_decoder(&(lzma->stream), UINT64_MAX, LZMA_TELL_UNSUPPORTED_CHECK | LZMA_CONCATENATED);
-
-   if(ret == LZMA_MEM_ERROR)
-      throw Memory_Exhaustion();
-   else if(ret != LZMA_OK)
-      throw Invalid_Argument("Bad setting in lzma_stream_decoder");
-   }
-
-/*
-* Decompress Input with Lzma
-*/
-void Lzma_Decompression::write(const byte input_arr[], size_t length)
-   {
-   if(length) no_writes = false;
-
-   const uint8_t* input = reinterpret_cast<const uint8_t*>(input_arr);
-
-   lzma->stream.next_in = input;
-   lzma->stream.avail_in = length;
-
-   while(lzma->stream.avail_in != 0)
-      {
-      lzma->stream.next_out = reinterpret_cast<uint8_t*>(&buffer[0]);
-      lzma->stream.avail_out = buffer.size();
-
-      lzma_ret ret = lzma_code(&(lzma->stream), LZMA_RUN);
-
-      if(ret != LZMA_OK && ret != LZMA_STREAM_END)
-         {
-         clear();
-         if(ret == LZMA_DATA_ERROR)
-            throw Decoding_Error("Lzma_Decompression: Data integrity error");
-         else if(ret == LZMA_MEM_ERROR)
-            throw Memory_Exhaustion();
-         else
-            throw std::runtime_error("Lzma decompression: Unknown error");
-         }
-
-      send(&buffer[0], buffer.size() - lzma->stream.avail_out);
-
-      if(ret == LZMA_STREAM_END)
-         {
-         size_t read_from_block = length - lzma->stream.avail_in;
-         start_msg();
-
-         lzma->stream.next_in = input + read_from_block;
-         lzma->stream.avail_in = length - read_from_block;
-
-         input += read_from_block;
-         length -= read_from_block;
-         }
-      }
-   }
-
-/*
-* Finish Decompressing with Lzma
-*/
-void Lzma_Decompression::end_msg()
-   {
-   if(no_writes) return;
-   lzma->stream.next_in = 0;
-   lzma->stream.avail_in = 0;
-
-   int ret = LZMA_OK;
-
-   while(ret != LZMA_STREAM_END)
-      {
-      lzma->stream.next_out = reinterpret_cast<uint8_t*>(&buffer[0]);
-      lzma->stream.avail_out = buffer.size();
-      ret = lzma_code(&(lzma->stream), LZMA_FINISH);
-
-      if(ret != LZMA_OK && ret != LZMA_STREAM_END)
-         {
-         clear();
-         throw Decoding_Error("Lzma_Decompression: Error finalizing");
-         }
-
-      send(&buffer[0], buffer.size() - lzma->stream.avail_out);
-      }
-
-   clear();
-   }
-
-/*
-* Clean up Decompression Context
-*/
-void Lzma_Decompression::clear()
-   {
-   zeroise(buffer);
-
-   no_writes = true;
-
-   if(lzma)
-      {
-      lzma_end(&(lzma->stream));
-      delete lzma;
-      lzma = 0;
-      }
-   }
-
-}
diff --git a/src/lib/filters/compression/lzma/lzma.h b/src/lib/filters/compression/lzma/lzma.h
deleted file mode 100644
index 33e12d580..000000000
--- a/src/lib/filters/compression/lzma/lzma.h
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
-* Lzma Compressor
-* (C) 2001 Peter J Jones
-*     2001-2007 Jack Lloyd
-*     2012 Vojtech Kral
-*
-* Distributed under the terms of the Botan license
-*/
-
-#ifndef BOTAN_LZMA_H__
-#define BOTAN_LZMA_H__
-
-#include <botan/filter.h>
-
-namespace Botan {
-
-/**
-* Lzma Compression Filter
-*/
-class BOTAN_DLL Lzma_Compression : public Filter
-   {
-   public:
-      std::string name() const { return "Lzma_Compression"; }
-
-      void write(const byte input[], size_t length);
-      void start_msg();
-      void end_msg();
-
-      /**
-      * Flush the compressor
-      */
-      void flush();
-
-      /**
-      * @param level how much effort to use on compressing (0 to 9);
-      *        higher levels are slower but tend to give better
-      *        compression
-      */
-      Lzma_Compression(size_t level = 6);
-
-      ~Lzma_Compression() { clear(); }
-   private:
-      void clear();
-      const size_t level;
-
-      secure_vector<byte> buffer;
-      class Lzma_Stream* lzma;
-   };
-
-/**
-* Lzma Decompression Filter
-*/
-class BOTAN_DLL Lzma_Decompression : public Filter
-   {
-   public:
-      std::string name() const { return "Lzma_Decompression"; }
-
-      void write(const byte input[], size_t length);
-      void start_msg();
-      void end_msg();
-
-      Lzma_Decompression();
-      ~Lzma_Decompression() { clear(); }
-   private:
-      void clear();
-
-      secure_vector<byte> buffer;
-      class Lzma_Stream* lzma;
-      bool no_writes;
-   };
-
-}
-
-#endif
diff --git a/src/lib/filters/compression/zlib/info.txt b/src/lib/filters/compression/zlib/info.txt
deleted file mode 100644
index d05e407d2..000000000
--- a/src/lib/filters/compression/zlib/info.txt
+++ /dev/null
@@ -1,11 +0,0 @@
-define COMPRESSOR_ZLIB 20131128
-
-load_on request
-
-<libs>
-all -> z
-</libs>
-
-<requires>
-filters
-</requires>
diff --git a/src/lib/filters/compression/zlib/zlib.cpp b/src/lib/filters/compression/zlib/zlib.cpp
deleted file mode 100644
index 4a0b2c8b4..000000000
--- a/src/lib/filters/compression/zlib/zlib.cpp
+++ /dev/null
@@ -1,318 +0,0 @@
-/*
-* Zlib Compressor
-* (C) 2001 Peter J Jones
-*     2001-2007 Jack Lloyd
-*     2006 Matt Johnston
-*
-* Distributed under the terms of the Botan license
-*/
-
-#include <botan/zlib.h>
-#include <botan/exceptn.h>
-
-#include <cstring>
-#include <cstdlib>
-#include <map>
-#include <zlib.h>
-
-namespace Botan {
-
-namespace {
-
-/*
-* Allocation Information for Zlib
-*/
-class Zlib_Alloc_Info
-   {
-   public:
-      std::map<void*, size_t> current_allocs;
-   };
-
-/*
-* Allocation Function for Zlib
-*/
-void* zlib_malloc(void* info_ptr, unsigned int n, unsigned int size)
-   {
-   Zlib_Alloc_Info* info = static_cast<Zlib_Alloc_Info*>(info_ptr);
-
-   const size_t total_sz = n * size;
-
-   void* ptr = std::malloc(total_sz);
-   info->current_allocs[ptr] = total_sz;
-   return ptr;
-   }
-
-/*
-* Allocation Function for Zlib
-*/
-void zlib_free(void* info_ptr, void* ptr)
-   {
-   Zlib_Alloc_Info* info = static_cast<Zlib_Alloc_Info*>(info_ptr);
-   auto i = info->current_allocs.find(ptr);
-   if(i == info->current_allocs.end())
-      throw Invalid_Argument("zlib_free: Got pointer not allocated by us");
-
-   std::memset(ptr, 0, i->second);
-   std::free(ptr);
-   }
-
-}
-
-/**
-* Wrapper Type for Zlib z_stream
-*/
-class Zlib_Stream
-   {
-   public:
-      /**
-      * Underlying stream
-      */
-      z_stream stream;
-
-      /**
-      * Constructor
-      */
-      Zlib_Stream()
-         {
-         std::memset(&stream, 0, sizeof(z_stream));
-         stream.zalloc = zlib_malloc;
-         stream.zfree = zlib_free;
-         stream.opaque = new Zlib_Alloc_Info;
-         }
-
-      /**
-      * Destructor
-      */
-      ~Zlib_Stream()
-         {
-         Zlib_Alloc_Info* info = static_cast<Zlib_Alloc_Info*>(stream.opaque);
-         delete info;
-         std::memset(&stream, 0, sizeof(z_stream));
-         }
-   };
-
-/*
-* Zlib_Compression Constructor
-*/
-Zlib_Compression::Zlib_Compression(size_t l, bool raw_deflate) :
-   level((l >= 9) ? 9 : l),
-   raw_deflate(raw_deflate),
-   buffer(DEFAULT_BUFFERSIZE),
-   zlib(0)
-   {
-   }
-
-/*
-* Start Compressing with Zlib
-*/
-void Zlib_Compression::start_msg()
-   {
-   clear();
-   zlib = new Zlib_Stream;
-
-   int res = deflateInit2(&(zlib->stream),
-                          level,
-                          Z_DEFLATED,
-                          (raw_deflate ? -15 : 15),
-                          8,
-                          Z_DEFAULT_STRATEGY);
-
-   if(res == Z_STREAM_ERROR)
-      throw Invalid_Argument("Bad setting in deflateInit2");
-   else if(res != Z_OK)
-      throw Memory_Exhaustion();
-   }
-
-/*
-* Compress Input with Zlib
-*/
-void Zlib_Compression::write(const byte input[], size_t length)
-   {
-   zlib->stream.next_in = static_cast<Bytef*>(const_cast<byte*>(input));
-   zlib->stream.avail_in = length;
-
-   while(zlib->stream.avail_in != 0)
-      {
-      zlib->stream.next_out = static_cast<Bytef*>(&buffer[0]);
-      zlib->stream.avail_out = buffer.size();
-      deflate(&(zlib->stream), Z_NO_FLUSH);
-      send(&buffer[0], buffer.size() - zlib->stream.avail_out);
-      }
-   }
-
-/*
-* Finish Compressing with Zlib
-*/
-void Zlib_Compression::end_msg()
-   {
-   zlib->stream.next_in = 0;
-   zlib->stream.avail_in = 0;
-
-   int rc = Z_OK;
-   while(rc != Z_STREAM_END)
-      {
-      zlib->stream.next_out = reinterpret_cast<Bytef*>(&buffer[0]);
-      zlib->stream.avail_out = buffer.size();
-
-      rc = deflate(&(zlib->stream), Z_FINISH);
-      send(&buffer[0], buffer.size() - zlib->stream.avail_out);
-      }
-
-   clear();
-   }
-
-/*
-* Flush the Zlib Compressor
-*/
-void Zlib_Compression::flush()
-   {
-   zlib->stream.next_in = 0;
-   zlib->stream.avail_in = 0;
-
-   while(true)
-      {
-      zlib->stream.avail_out = buffer.size();
-      zlib->stream.next_out = reinterpret_cast<Bytef*>(&buffer[0]);
-
-      deflate(&(zlib->stream), Z_FULL_FLUSH);
-      send(&buffer[0], buffer.size() - zlib->stream.avail_out);
-
-      if(zlib->stream.avail_out == buffer.size())
-        break;
-      }
-   }
-
-/*
-* Clean up Compression Context
-*/
-void Zlib_Compression::clear()
-   {
-   zeroise(buffer);
-
-   if(zlib)
-      {
-      deflateEnd(&(zlib->stream));
-      delete zlib;
-      zlib = 0;
-      }
-   }
-
-/*
-* Zlib_Decompression Constructor
-*/
-Zlib_Decompression::Zlib_Decompression(bool raw_deflate) :
-   raw_deflate(raw_deflate),
-   buffer(DEFAULT_BUFFERSIZE),
-   zlib(0),
-   no_writes(true)
-   {
-   }
-
-/*
-* Start Decompressing with Zlib
-*/
-void Zlib_Decompression::start_msg()
-   {
-   clear();
-   zlib = new Zlib_Stream;
-
-   if(inflateInit2(&(zlib->stream), (raw_deflate ? -15 : 15)) != Z_OK)
-      throw Memory_Exhaustion();
-   }
-
-/*
-* Decompress Input with Zlib
-*/
-void Zlib_Decompression::write(const byte input_arr[], size_t length)
-   {
-   if(length) no_writes = false;
-
-   // non-const needed by zlib api :(
-   Bytef* input = reinterpret_cast<Bytef*>(const_cast<byte*>(input_arr));
-
-   zlib->stream.next_in = input;
-   zlib->stream.avail_in = length;
-
-   while(zlib->stream.avail_in != 0)
-      {
-      zlib->stream.next_out = reinterpret_cast<Bytef*>(&buffer[0]);
-      zlib->stream.avail_out = buffer.size();
-
-      int rc = inflate(&(zlib->stream), Z_SYNC_FLUSH);
-
-      if(rc != Z_OK && rc != Z_STREAM_END)
-         {
-         clear();
-         if(rc == Z_DATA_ERROR)
-            throw Decoding_Error("Zlib_Decompression: Data integrity error");
-         else if(rc == Z_NEED_DICT)
-            throw Decoding_Error("Zlib_Decompression: Need preset dictionary");
-         else if(rc == Z_MEM_ERROR)
-            throw Memory_Exhaustion();
-         else
-            throw std::runtime_error("Zlib decompression: Unknown error");
-         }
-
-      send(&buffer[0], buffer.size() - zlib->stream.avail_out);
-
-      if(rc == Z_STREAM_END)
-         {
-         size_t read_from_block = length - zlib->stream.avail_in;
-         start_msg();
-
-         zlib->stream.next_in = input + read_from_block;
-         zlib->stream.avail_in = length - read_from_block;
-
-         input += read_from_block;
-         length -= read_from_block;
-         }
-      }
-   }
-
-/*
-* Finish Decompressing with Zlib
-*/
-void Zlib_Decompression::end_msg()
-   {
-   if(no_writes) return;
-   zlib->stream.next_in = 0;
-   zlib->stream.avail_in = 0;
-
-   int rc = Z_OK;
-
-   while(rc != Z_STREAM_END)
-      {
-      zlib->stream.next_out = reinterpret_cast<Bytef*>(&buffer[0]);
-      zlib->stream.avail_out = buffer.size();
-      rc = inflate(&(zlib->stream), Z_SYNC_FLUSH);
-
-      if(rc != Z_OK && rc != Z_STREAM_END)
-         {
-         clear();
-         throw Decoding_Error("Zlib_Decompression: Error finalizing");
-         }
-
-      send(&buffer[0], buffer.size() - zlib->stream.avail_out);
-      }
-
-   clear();
-   }
-
-/*
-* Clean up Decompression Context
-*/
-void Zlib_Decompression::clear()
-   {
-   zeroise(buffer);
-
-   no_writes = true;
-
-   if(zlib)
-      {
-      inflateEnd(&(zlib->stream));
-      delete zlib;
-      zlib = 0;
-      }
-   }
-
-}
diff --git a/src/lib/filters/compression/zlib/zlib.h b/src/lib/filters/compression/zlib/zlib.h
deleted file mode 100644
index c4d21a250..000000000
--- a/src/lib/filters/compression/zlib/zlib.h
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
-* Zlib Compressor
-* (C) 2001 Peter J Jones
-*     2001-2007 Jack Lloyd
-*
-* Distributed under the terms of the Botan license
-*/
-
-#ifndef BOTAN_ZLIB_H__
-#define BOTAN_ZLIB_H__
-
-#include <botan/filter.h>
-
-namespace Botan {
-
-/**
-* Zlib Compression Filter
-*/
-class BOTAN_DLL Zlib_Compression : public Filter
-   {
-   public:
-      std::string name() const { return "Zlib_Compression"; }
-
-      void write(const byte input[], size_t length);
-      void start_msg();
-      void end_msg();
-
-      /**
-      * Flush the compressor
-      */
-      void flush();
-
-      /**
-      * @param level how much effort to use on compressing (0 to 9);
-      *        higher levels are slower but tend to give better
-      *        compression
-      * @param raw_deflate if true no zlib header/trailer will be used
-      */
-      Zlib_Compression(size_t level = 6,
-                       bool raw_deflate = false);
-
-      ~Zlib_Compression() { clear(); }
-   private:
-      void clear();
-      const size_t level;
-      const bool raw_deflate;
-
-      secure_vector<byte> buffer;
-      class Zlib_Stream* zlib;
-   };
-
-/**
-* Zlib Decompression Filter
-*/
-class BOTAN_DLL Zlib_Decompression : public Filter
-   {
-   public:
-      std::string name() const { return "Zlib_Decompression"; }
-
-      void write(const byte input[], size_t length);
-      void start_msg();
-      void end_msg();
-
-      Zlib_Decompression(bool raw_deflate = false);
-      ~Zlib_Decompression() { clear(); }
-   private:
-      void clear();
-
-      const bool raw_deflate;
-
-      secure_vector<byte> buffer;
-      class Zlib_Stream* zlib;
-      bool no_writes;
-   };
-
-}
-
-#endif
diff --git a/src/lib/filters/info.txt b/src/lib/filters/info.txt
index 4145af34a..3318257d1 100644
--- a/src/lib/filters/info.txt
+++ b/src/lib/filters/info.txt
@@ -4,6 +4,7 @@ define FILTERS 20131128
 algo_filt.cpp
 basefilt.cpp
 buf_filt.cpp
+comp_filter.cpp
 data_snk.cpp
 data_src.cpp
 filter.cpp
@@ -20,6 +21,7 @@ transform_filter.cpp
 basefilt.h
 buf_filt.h
 data_snk.h
+comp_filter.h
 data_src.h
 filter.h
 filters.h