Merge pull request #6 from fast-pack/revert-5-update_build_1

Revert "Update build review"

Author: searchivarius

.github/workflows/buildwheels.yml

@@ -0,0 +1,63 @@
+language: cpp
+
+matrix:
+  fast_finish: true
+  include:
+  - os: linux
+    env: PYTHON=2.7
+  - os: linux
+    env: PYTHON=3
+  - os: osx
+    env: PYTHON=3.6
+  allow_failures:
+  - os: osx
+
+addons:
+  apt:
+    sources:
+    - ubuntu-toolchain-r-test
+    packages:
+    - g++-4.9
+
+before_install:
+- |
+  PIP=pip
+  PY=python
+  if [ "$TRAVIS_OS_NAME" = "linux" ]; then export CXX=g++-4.9 CC=gcc-4.9; fi
+  if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
+    brew update
+    brew install gcc
+    PIP=pip2
+    PY=python2
+    if  [ "${PYTHON:0:1}" = "3" ]; then
+      brew upgrade python
+      brew install sashkab/python/python$PYTHON
+      PIP=pip3
+      PY=python3
+    fi
+  fi
+  $PIP install --user virtualenv
+  virtualenv -p $PY venv
+  source venv/bin/activate
+  $PIP install --upgrade pip
+  $PIP install -qq setuptools
+
+install:
+- |
+  travis_wait travis_retry $PIP install -r python_bindings/requirements.txt
+  travis_retry cd python_bindings && $PY setup.py build install && cd ..
+
+script:
+- $PY --version
+- $PIP --version
+- |
+  set -e
+  cd python_bindings
+  $PY setup.py test
+  cd ..
+
+cache:
+  - apt
+  - directories:
+    - $HOME/.cache/pip
+

.gitignore

@@ -3,13 +3,13 @@
 # Apache License Version 2.0 http://www.apache.org/licenses/.
 #
 # This is a modified version of the CMakeFile
-# for the project https://github.com/lemire/FastPFor
-# This file can be used to
+# for the project https://github.com/lemire/FastPFor 
+# This file can be used to 
 #   1) Build a library separately. Then, this library
 #      will be re-used by the Python build process
 #   2) Build and run unit and performance tests.
 #
-cmake_minimum_required(VERSION 3.10)
+cmake_minimum_required(VERSION 2.8.7)
 if (NOT CMAKE_BUILD_TYPE)
 		message(STATUS "No build type selected, default to Release")
 		set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build." FORCE)
@@ -21,7 +21,7 @@ include(AppendCompilerFlags)
 
 
 project(PyFastPFor CXX C)
-set(PROJECT_URL "https://github.com/fast-pack/PyFastPFOR")
+set(PROJECT_URL "https://github.com/searchivarius/PyFastPFOR")
 set(PROJECT_DESCRIPTION "Python bindings for the FastPFOR C++ library: Fast integer compression")
 include(DetectCPUFeatures)
 #
@@ -54,10 +54,8 @@ MESSAGE( STATUS "CMAKE_C_COMPILER: " ${CMAKE_C_COMPILER} )
 MESSAGE( STATUS "CXX_COMPILER_VERSION: " ${CXX_COMPILER_VERSION} )
 if( SUPPORT_SSE42 )
     MESSAGE( STATUS "SSE 4.2 support detected" )
-elseif( SUPPORT_NEON )
-    MESSAGE( STATUS "ARM NEON detected: SSE intrinsics are emulated via fastpfor_neon.h" )
 else()
-    MESSAGE( STATUS "Neither SSE 4.2 nor ARM NEON support detected" )
+    MESSAGE( STATUS "SSE 4.2 support not detected" )
 endif()
 
 if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
@@ -81,21 +79,14 @@ elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel")
     set (CMAKE_CXX_FLAGS_DEBUG   "-Wall -ggdb   -std=c++11 -DHAVE_CXX0X  -march=native")
     set (CMAKE_C_FLAGS_RELEASE "-Wall -Ofast  -DNDEBUG -std=c99  -march=native")
     set (CMAKE_C_FLAGS_DEBUG   "-Wall -ggdb   -std=c99  -march=native")
-elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang")
+elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
     if (CXX_COMPILER_VERSION VERSION_LESS 4.2.1)
         message(STATUS  "Clang version must be at least 4.2.1!" )
     endif()
-    # -msse4.1 is x86-only and rejected on ARM; -march=native already enables the
-    # available SIMD (SSE/AVX on x86, NEON on ARM), so only add it off ARM.
-    if (SUPPORT_NEON)
-        set (SIMD_FLAGS "-march=native")
-    else()
-        set (SIMD_FLAGS "-msse4.1 -march=native")
-    endif()
-    set (CMAKE_CXX_FLAGS_RELEASE "-Wall -Wcast-align -O3 -DNDEBUG -std=c++11 -DHAVE_CXX0X ${SIMD_FLAGS}")
-    set (CMAKE_CXX_FLAGS_DEBUG   "-Wall -Wcast-align -ggdb  -std=c++11 -DHAVE_CXX0X ${SIMD_FLAGS}")
-    set (CMAKE_C_FLAGS_RELEASE "-Wall -Wcast-align -O3 -DNDEBUG -std=c99 ${SIMD_FLAGS}")
-    set (CMAKE_C_FLAGS_DEBUG   "-Wall -Wcast-align -ggdb  -std=c99 ${SIMD_FLAGS}")
+    set (CMAKE_CXX_FLAGS_RELEASE "-Wall -Wcast-align -O3 -DNDEBUG -std=c++11 -DHAVE_CXX0X -msse4.1 -march=native")
+    set (CMAKE_CXX_FLAGS_DEBUG   "-Wall -Wcast-align -ggdb  -std=c++11 -DHAVE_CXX0X -msse4.1 -march=native")
+    set (CMAKE_C_FLAGS_RELEASE "-Wall -Wcast-align -O3 -DNDEBUG -std=c99 -msse4.1 -march=native")
+    set (CMAKE_C_FLAGS_DEBUG   "-Wall -Wcast-align -ggdb  -std=c99 -msse4.1 -march=native")
 elseif(WIN32)
     # TODO add support for later versions?
     if(NOT MSVC12)

.travis.yml

@@ -19,7 +19,7 @@ or via pip:
 ```
 pip install pyfastpfor
 ```
-The bindings build with GCC or Clang, on both x86-64 (SSE/AVX) and ARM/aarch64 (NEON, including Apple Silicon). You may also need to install Python dev-files. On Ubuntu, for Python 3 you can do it as follows:
+Due to some compilation quirks this currently seem to work with GCC only. I will fix it in some not so distant future. You may also need to install Python dev-files. On Ubuntu, for Python 3 you can do it as follows:
 
 ```
 sudo apt-get install python3-dev
@@ -28,7 +28,7 @@ sudo apt-get install python3-dev
 
 # Documentation
 
-The library supports all the codecs implemented in the original [FastPFor](https://github.com/lemire/FastPFor) library (v0.5.0). To get a list of codecs, use the function ``getCodecList``. 
+The library supports all the codecs implemented in the original [FastPFor](https://github.com/lemire/FastPFor) library by July 2023. To get a list of codecs, use the function ``getCodecList``. 
 
 Typical light-weight compression does not take context into account and, consequently, works well only for small integers. When integers are large, data differencing is a common trick to make integers small. In particular, we often deal with sorted lists of integers, which can be represented by differences between neighboring numbers. 
 

CMakeLists.txt

@@ -47,17 +47,7 @@ else()
 	check_cxx_source_compiles("${AVXPROG}" SUPPORT_AVX)
 	set(CMAKE_REQUIRED_FLAGS "-march=native -mavx2")
 	check_cxx_source_compiles("${AVX2PROG}" SUPPORT_AVX2)
-endif()
+endif()	
 
 set(CMAKE_REQUIRED_FLAGS ${OLD_CMAKE_REQUIRED_FLAGS})
-
-# On ARM the SSE intrinsics used throughout FastPFor are provided by the
-# fastpfor_neon.h shim (mapped onto NEON), so there is no SSE4.2 support but the
-# code still compiles. Flag ARM builds so the SIMD-specific compiler options
-# (e.g. -msse4.1) can be skipped.
-if (CMAKE_SYSTEM_PROCESSOR MATCHES "arm64" OR CMAKE_SYSTEM_PROCESSOR MATCHES "arm"
-    OR CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64"
-    OR CMAKE_GENERATOR_PLATFORM MATCHES "ARM64" OR CMAKE_GENERATOR_PLATFORM MATCHES "ARM")
-	set(SUPPORT_NEON ON)
-endif ()
 

README.md

@@ -2,7 +2,7 @@
  * This code is released under the
  * Apache License Version 2.0 http://www.apache.org/licenses/.
  */
-#if (!defined(__SSSE3__) && !(defined(_MSC_VER) && defined(__AVX__))) && !(defined(__ARM_NEON) || defined(__aarch64__))
+#if !defined(__SSSE3__) && !(defined(_MSC_VER) && defined(__AVX__))
 #ifndef _MSC_VER
 #pragma message                                                                \
     "Disabling varintg8iu due to lack of SSSE3 support, try adding -mssse3 or the equivalent on your compiler"
@@ -12,12 +12,7 @@
 #else
 #ifndef VARINTG8IU_H__
 #define VARINTG8IU_H__
-#if defined(__GNUC__) && (defined(__x86_64__) || defined(__i386__))
 #include <emmintrin.h>
-#elif defined(__aarch64__)
-/* GCC-compatible compiler, targeting ARM with native NEON */
-#include "fastpfor_neon.h"
-#endif
 #include "codecs.h"
 #ifdef __GNUC__
 #define PREDICT_FALSE(x) (__builtin_expect(x, 0))

cmake_modules/DetectCPUFeatures.cmake

@@ -286,7 +286,6 @@ class ByteAlignedPacking : public IntegerCODEC {
 
   const uint32_t *decodeArray(const uint32_t *in, const size_t length,
                               uint32_t *out, size_t &nvalue) {
-    (void)length;
     const uint32_t actuallength = *in++;
     const uint8_t *inbyte = reinterpret_cast<const uint8_t *>(in);
     const uint32_t *const initout(out);

headers/VarIntG8IU.h

@@ -21,54 +21,15 @@ class CODECFactory {
 public:
   CODECFactory();
 
-  std::vector<std::shared_ptr<IntegerCODEC>> allSchemes() const;
+  std::vector<std::shared_ptr<IntegerCODEC>> allSchemes();
 
-  std::vector<std::string> allNames() const;
+  std::vector<std::string> allNames();
 
-  std::shared_ptr<IntegerCODEC> const& getFromName(std::string name) const;
+  std::shared_ptr<IntegerCODEC> &getFromName(std::string name);
 private:
   CodecMap scodecmap;
 };
 
-std::unique_ptr<IntegerCODEC> fastbinarypacking8_codec();
-std::unique_ptr<IntegerCODEC> fastbinarypacking16_codec();
-std::unique_ptr<IntegerCODEC> fastbinarypacking32_codec();
-std::unique_ptr<IntegerCODEC> BP32_codec();
-std::unique_ptr<IntegerCODEC> vsencoding_codec();
-std::unique_ptr<IntegerCODEC> fastpfor128_codec();
-std::unique_ptr<IntegerCODEC> fastpfor256_codec();
-std::unique_ptr<IntegerCODEC> simdfastpfor128_codec();
-std::unique_ptr<IntegerCODEC> simdfastpfor256_codec();
-std::unique_ptr<IntegerCODEC> simplepfor_codec();
-std::unique_ptr<IntegerCODEC> simdsimplepfor_codec();
-std::unique_ptr<IntegerCODEC> pfor_codec();
-std::unique_ptr<IntegerCODEC> simdpfor_codec();
-std::unique_ptr<IntegerCODEC> pfor2008_codec();
-std::unique_ptr<IntegerCODEC> simdnewpfor_codec();
-std::unique_ptr<IntegerCODEC> newpfor_codec();
-std::unique_ptr<IntegerCODEC> optpfor_codec();
-std::unique_ptr<IntegerCODEC> simdoptpfor_codec();
-std::unique_ptr<IntegerCODEC> varint_codec();
-std::unique_ptr<IntegerCODEC> vbyte_codec();
-std::unique_ptr<IntegerCODEC> maskedvbyte_codec();
-std::unique_ptr<IntegerCODEC> streamvbyte_codec();
-std::unique_ptr<IntegerCODEC> varintgb_codec();
-std::unique_ptr<IntegerCODEC> simple16_codec();
-std::unique_ptr<IntegerCODEC> simple9_codec();
-std::unique_ptr<IntegerCODEC> simple9_rle_codec();
-std::unique_ptr<IntegerCODEC> simple8b_codec();
-std::unique_ptr<IntegerCODEC> simple8b_rle_codec();
-#ifdef VARINTG8IU_H__
-std::unique_ptr<IntegerCODEC> varintg8iu_codec();
-#endif
-#ifdef USESNAPPY
-std::unique_ptr<IntegerCODEC> snappy_codec();
-#endif
-std::unique_ptr<IntegerCODEC> simdbinarypacking_codec();
-std::unique_ptr<IntegerCODEC> simdgroupsimple_codec();
-std::unique_ptr<IntegerCODEC> simdgroupsimple_ringbuf_codec();
-std::unique_ptr<IntegerCODEC> copy_codec();
-
 } // namespace FastPForLib
 
 #endif /* CODECFACTORY_H_ */

headers/blockpacking.h

@@ -53,10 +53,6 @@ class IntegerCODEC {
    * of the variable nvalue gets updated with the number actually use
    * (if nvalue exceeds the original value, there might be a buffer
    * overrun).
-   *
-   * NOTE: Decoding can be performed with an unknown input length. This
-   * case is indicated by a length of 0; however, nvalue must be provided
-   * in order for the decoder knows how many values to decode.
    */
   virtual const uint32_t *decodeArray(const uint32_t *in, const size_t length,
                                       uint32_t *out, size_t &nvalue) = 0;