Merge branch 'main' into dependabot/pip/pyarrow-gte-15-and-lt-25

Author: timkpaine

.github/workflows/build.yml

@@ -756,7 +756,7 @@ jobs:
           run: ls -R ./dist
 
         - name: Publish to github releases
-          uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3.0.2
+          uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3.0.3
           with:
             draft: true
             generate_release_notes: true

conda/dev-environment-unix.yml

@@ -17,7 +17,7 @@ dependencies:
  - flex
  - graphviz
  - gtest
- - httpx>=0.20,<1
+ - httpx2>=2,<3
  - libarrow<25
  - libboost>=1.80.0
  - libboost-headers>=1.80.0
@@ -47,7 +47,7 @@ dependencies:
  - rapidjson
  - requests
  - ruamel.yaml
- - ruff>=0.9,<0.15
+ - ruff>=0.9,<0.16
  - scikit-build
  - setuptools>=69,<74
  - sqlalchemy

conda/dev-environment-win.yml

@@ -17,7 +17,7 @@ dependencies:
 #  - flex  # not available on windows
  - graphviz
  - gtest
- - httpx>=0.20,<1
+ - httpx2>=2,<3
  - libarrow<25
  - libboost>=1.80.0
  - libboost-headers>=1.80.0
@@ -47,7 +47,7 @@ dependencies:
  - rapidjson
  - requests
  - ruamel.yaml
- - ruff>=0.9,<0.15
+ - ruff>=0.9,<0.16
  - scikit-build
  - setuptools>=69,<74
  - sqlalchemy

cpp/csp/core/DynamicBitSet.h

@@ -155,19 +155,12 @@ class DynamicBitSet
 private:
     using nbit_type = uint8_t;
 
-#ifndef WIN32
-#define CLZ_CONSTEXPR constexpr
-#else
-#define CLZ_CONSTEXPR 
-#endif
-
     template<typename value_type, 
         std::enable_if_t<std::is_unsigned<value_type>::value, bool> = true>
     static constexpr nbit_type nbits() { return sizeof( value_type ) * 8; }
 
     template<typename U, std::enable_if_t<std::is_unsigned<U>::value, bool> = true>
-    static CLZ_CONSTEXPR nbit_type log2( U n )           { return nbits<uint32_t>() - clz(static_cast<uint32_t>( n )) - 1; } //upcast to 32 bit to avoid truncation for log2
-    static CLZ_CONSTEXPR nbit_type log2(uint64_t n)      { return nbits<uint64_t>() - clz(n) - 1; }
+    static constexpr nbit_type log2( U n )           { return nbits<U>() - clz( n ) - 1; }
 
     static constexpr node_type mask( nbit_type bitIndex )  { return ( node_type )1 << bitIndex; }
 
@@ -182,7 +175,7 @@ class DynamicBitSet
     }
 
     static constexpr nbit_type _bits               = nbits<NodeT>();
-    static inline CLZ_CONSTEXPR nbit_type _logBits = log2( _bits );
+    static inline constexpr nbit_type _logBits = log2( _bits );
 
     node_type * m_nodes;
     index_type  m_size;

cpp/csp/core/Platform.h

@@ -1,5 +1,6 @@
 #ifndef _IN_CSP_CORE_PLATFORM_H
 #define _IN_CSP_CORE_PLATFORM_H
+#include <bit>
 #include <type_traits>
 #include <stdint.h>
 #include <time.h>
@@ -47,47 +48,11 @@ inline tm * localtime_r( const time_t * timep, tm * result )
 inline int nanosleep(const timespec* req, timespec* rem)
 {
     assert(rem == nullptr);
-    int64_t millis = req->tv_sec * 1000 + req->tv_nsec * 1000000;
+    int64_t millis = req->tv_sec * 1000 + req->tv_nsec / 1000000;
     Sleep(millis);
     return 0;
 }
 
-inline uint8_t clz(uint64_t n)
-{
-    unsigned long index = 0;
-    if (_BitScanReverse64(&index, n))
-	    return 64 - index - 1;
-    return 0;
-}
-
-inline uint8_t clz(uint32_t n)
-{
-    unsigned long index = 0;
-    if (_BitScanReverse(&index, n))
-	    return 32 - index - 1;
-    return 0;
-}
-
-inline uint8_t clz(uint16_t n) { return clz(static_cast<uint32_t>(n)) - 16; }
-inline uint8_t clz(uint8_t n)  { return clz(static_cast<uint32_t>(n)) - 24; }
-
-template<typename U, std::enable_if_t<std::is_unsigned<U>::value, bool> = true>
-inline uint8_t ffs(U n)
-{ 
-    unsigned long index = 0;
-    if (_BitScanForward(&index, n))
-	    return index + 1;
-    return 0;
-}
-
-inline uint8_t ffs(uint64_t n)
-{
-    unsigned long index = 0;
-    if (_BitScanForward64(&index, n))
-	    return index + 1;
-    return 0;
-}
-
 #else
 
 #define CSPIMPL_EXPORT
@@ -100,19 +65,14 @@ inline uint8_t ffs(uint64_t n)
 
 #define NO_INLINE  __attribute__ ((noinline))
 
-inline constexpr uint8_t clz(uint32_t n) { return __builtin_clz(n); }
-inline constexpr uint8_t clz(uint64_t n) { return __builtin_clzl(n); }
+#endif
 
 // clz (count leading zeros) returns number of leading zeros before MSB (i.e. clz(00110..) = 2 )
-// __builtin_clz auto-promotes to 32-bits: need to subtract off extra leading zeros
-inline constexpr uint8_t clz(uint16_t n) { return clz(static_cast<uint32_t>(n)) - 16; }
-inline constexpr uint8_t clz(uint8_t n)  { return clz(static_cast<uint32_t>(n)) - 24; }
+template<typename U, std::enable_if_t<std::is_unsigned<U>::value, bool> = true>
+inline constexpr uint8_t clz( U n )        { return std::countl_zero(n); }
 
 // ffs (find first set) returns offset of first set bit (i.e. ffs(..0110) = 2 ), with ffs(0) = 0
 template<typename U, std::enable_if_t<std::is_unsigned<U>::value, bool> = true>
-inline constexpr uint8_t ffs( U n )        { return __builtin_ffs(n); }
-inline constexpr uint8_t ffs( uint64_t n ) { return __builtin_ffsl(n); }
-
-#endif
+inline constexpr uint8_t ffs( U n )        { return n ? std::countr_zero(n) + 1 : 0; }
 
 #endif

cpp/csp/python/Conversions.h

@@ -446,7 +446,7 @@ inline PyObject * toPython( const CspEnum & e, const CspType & type )
     auto & enumType = static_cast<const CspEnumType&>( type );
     const auto * emeta = static_cast<const DialectCspEnumMeta*>( enumType.meta().get() );
 
-    PyObject * obj = emeta -> pyMeta() -> toPyEnum( e );
+    PyObject * obj = emeta -> toPyEnum( e.value() );
     if( !obj ) [[unlikely]]
         CSP_THROW( ValueError, e.value() << " is not a valid value on csp.enum type " << emeta -> name() );
     return obj;
@@ -457,11 +457,13 @@ inline CspEnum fromPython( PyObject * o, const CspType & type )
 {
     assert( type.type() == CspType::Type::ENUM );
 
-    if( !PyType_IsSubtype( Py_TYPE( o ), &PyCspEnum::PyType ) ||
-        static_cast<PyCspEnum *>( o ) -> meta() != static_cast<const CspEnumType &>( type ).meta().get() )
-        CSP_THROW( TypeError, "Invalid enum type, expected enum type " << static_cast<const CspEnumType &>( type ).meta() -> name() << " got " << Py_TYPE( o ) -> tp_name );
-
-    return static_cast<PyCspEnum *>( o ) -> enum_;
+    auto & enumType = static_cast<const CspEnumType&>( type );
+    const auto * emeta = static_cast<const DialectCspEnumMeta*>( enumType.meta().get() );
+    
+    if( !PyObject_IsInstance( o, ( PyObject * ) emeta -> pyType().get() ) )
+        CSP_THROW( TypeError, "Invalid enum type, expected enum type " << emeta -> pyType() -> tp_name << " got " << Py_TYPE( o ) -> tp_name );
+    
+    return static_cast<const CspEnumType &>( type ).meta() -> create( fromPython<int64_t>( o ) );
 }
 
 //TimeDelta

cpp/csp/python/CspTypeFactory.cpp

@@ -7,10 +7,23 @@
 namespace csp::python
 {
 
+CspTypeFactory::CspTypeFactory()
+{
+    PyObject *enum_mod = PyImport_ImportModule( "enum" );
+    m_intEnumPyType    = ( PyTypeObject * ) PyObject_GetAttrString( enum_mod, "IntEnum" );
+}
+
 CspTypeFactory & CspTypeFactory::instance()
 {
-    static CspTypeFactory s_instance;
-    return s_instance;
+    //We let this leak since some csp types ( ie CspEnum ) can hold a ref to DialectCspEnumMeta which holds Ptrs
+    //to python objects, which cant be destroyed statically after python interpreter is shutdown
+    static CspTypeFactory * s_instance = new CspTypeFactory();
+    return *s_instance;
+}
+
+bool CspTypeFactory::isCspEnumPyType( PyTypeObject * pyType )
+{
+    return PyType_IsSubtype( pyType, m_intEnumPyType );
 }
 
 CspTypePtr & CspTypeFactory::typeFromPyType( PyObject * pyTypeObj )
@@ -59,9 +72,9 @@ CspTypePtr & CspTypeFactory::typeFromPyType( PyObject * pyTypeObj )
             auto meta = ( ( PyStructMeta * ) pyType ) -> structMeta;
             rv.first -> second = std::make_shared<csp::CspStructType>( meta );
         }
-        else if( PyType_IsSubtype( pyType, &PyCspEnum::PyType ) )
+        else if( isCspEnumPyType( pyType ) )
         {
-            auto meta = ( ( PyCspEnumMeta * ) pyType ) -> enumMeta;
+            auto meta = createCspEnumMetaFromIntEnum( PyTypeObjectPtr::incref( pyType ) );
             rv.first -> second = std::make_shared<csp::CspEnumType>( meta );
         }
         else if( pyType == PyDateTimeAPI -> DateTimeType )
@@ -88,4 +101,30 @@ void CspTypeFactory::removeCachedType( PyTypeObject * pyType )
     m_cache.erase( pyType );
 }
 
+std::shared_ptr<CspEnumMeta> CspTypeFactory::createCspEnumMetaFromIntEnum( PyTypeObjectPtr pyIntEnumType )
+{
+    CspEnumMeta::ValueDef metadef;
+    
+    PyObjectPtr iter = PyObjectPtr::check( PyObject_GetIter( ( PyObject * ) pyIntEnumType.get() ) );
+    PyObject * member;
+    while( ( member = PyIter_Next( iter.get() ) ) != NULL )
+    {
+        PyObjectPtr name = PyObjectPtr::check( PyObject_GetAttrString( member, "name" ) );
+
+        const char * namestr = PyUnicode_AsUTF8( name.get() );
+        if( !namestr )
+            CSP_THROW( PythonPassthrough, "" );
+            
+        if( !PyLong_Check( member ) )
+            CSP_THROW( TypeError, "enum key " << namestr << " expected an integer got " << PyObjectPtr::incref( member ) );
+        
+        int64_t value = fromPython<int64_t>( member );
+        metadef[ namestr ] = value;
+        
+        Py_DECREF( member );
+    }
+    
+    return std::make_shared<DialectCspEnumMeta>( pyIntEnumType, pyIntEnumType -> tp_name, metadef );
+}
+
 }

cpp/csp/python/CspTypeFactory.h

@@ -3,6 +3,7 @@
 
 #include <csp/core/Platform.h>
 #include <csp/engine/CspType.h>
+#include <csp/python/PyObjectPtr.h>
 #include <unordered_map>
 #include <Python.h>
 
@@ -17,9 +18,17 @@ class CSPTYPESIMPL_EXPORT CspTypeFactory
     CspTypePtr & typeFromPyType( PyObject * );
     void removeCachedType( PyTypeObject * );
 
+    bool isCspEnumPyType( PyTypeObject * pyType );
+
 private:
     using Cache = std::unordered_map<PyTypeObject *, CspTypePtr>;
+
+    std::shared_ptr<CspEnumMeta> createCspEnumMetaFromIntEnum( PyTypeObjectPtr pyIntEnumType );
+
+    CspTypeFactory();
     Cache m_cache;
+
+    PyTypeObject * m_intEnumPyType;
 };
 
 }