Merge branch 'main' into tkp/autogen

Author: timkpaine

.github/workflows/build.yml

@@ -196,7 +196,7 @@ jobs:
             - ubuntu-24.04 # https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2404-Readme.md
             - macos-13 # https://github.com/actions/runner-images/blob/main/images/macos/macos-13-Readme.md
             - macos-14 # https://github.com/actions/runner-images/blob/main/images/macos/macos-14-arm64-Readme.md
-            - windows-2019 # https://github.com/actions/runner-images/blob/main/images/windows/Windows2019-Readme.md
+            - windows-2022 # https://github.com/actions/runner-images/blob/main/images/windows/Windows2022-Readme.md
           python-version:
             - "3.9"
             - "3.10"
@@ -250,15 +250,15 @@ jobs:
 
             # Avoid extra resources for windows build
             - is-full-run: false
-              os: windows-2019
+              os: windows-2022
               python-version: "3.9"
 
             - is-full-run: false
-              os: windows-2019
+              os: windows-2022
               python-version: "3.10"
 
             - is-full-run: false
-              os: windows-2019
+              os: windows-2022
               python-version: "3.11"
 
             # avoid unnecessary use of mac resources
@@ -328,14 +328,14 @@ jobs:
 
         ##########
         # Windows
-        - name: Python Build Steps (Windows 2019)
+        - name: Python Build Steps (Windows 2022)
           run: make dist-py-cibw
           env:
             CIBW_BUILD: "${{ matrix.cibuildwheel }}-win_amd64"
-            CSP_GENERATOR: "Visual Studio 16 2019"
+            CSP_GENERATOR: "Visual Studio 17 2022"
             VCPKG_DEFAULT_BINARY_CACHE: C:\\Users\\runneradmin\\AppData\\Local\\vcpkg_cache
             VCPKG_DOWNLOADS: C:\\Users\\runneradmin\\AppData\\Local\\vcpkg_download_cache
-          if: ${{ matrix.os == 'windows-2019' }}
+          if: ${{ matrix.os == 'windows-2022' }}
 
         ##########
         # Common
@@ -433,7 +433,7 @@ jobs:
             - ubuntu-24.04
             - macos-13
             - macos-14
-            - windows-2019
+            - windows-2022
           python-version:
             - 3.9
             - "3.10"
@@ -451,15 +451,15 @@ jobs:
 
             # Avoid extra resources for windows build
             - is-full-run: false
-              os: windows-2019
+              os: windows-2022
               python-version: "3.9"
 
             - is-full-run: false
-              os: windows-2019
+              os: windows-2022
               python-version: "3.10"
 
             - is-full-run: false
-              os: windows-2019
+              os: windows-2022
               python-version: "3.11"
 
             # avoid unnecessary use of mac resources
@@ -650,7 +650,7 @@ jobs:
       strategy:
         matrix:
           os:
-            - ubuntu-20.04
+            - ubuntu-24.04
           python-version:
             - 3.9
           package:

.github/workflows/conda.yml

@@ -38,7 +38,7 @@ jobs:
             - ubuntu-24.04
             - macos-14
             - macos-13
-            - windows-2019
+            - windows-2022
       runs-on: ${{ matrix.os }}
       steps:
         - name: Checkout
@@ -86,7 +86,7 @@ jobs:
 
         - name: Python Build Steps ( Windows )
           env:
-            CSP_GENERATOR: "Visual Studio 16 2019"
+            CSP_GENERATOR: "Visual Studio 17 2022"
           run: make build-conda
           shell: cmd /C call {0}
           if: ${{ runner.os == 'Windows' }}

.github/workflows/wiki-publish.yml

@@ -24,6 +24,6 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Upload Documentation to Wiki
-        uses: Andrew-Chen-Wang/github-wiki-action@v4
+        uses: Andrew-Chen-Wang/github-wiki-action@v5
         with:
           path: docs/wiki

CMakeLists.txt

@@ -2,7 +2,7 @@
 # Project Configuration #
 #########################
 cmake_minimum_required(VERSION 3.20.0)
-project(csp VERSION "0.0.11")
+project(csp VERSION "0.11.1")
 set(CMAKE_CXX_STANDARD 20)
 
 ###################################################################################################################################################
@@ -70,6 +70,7 @@ option(CSP_USE_LD_CLASSIC_MAC "On macOS, link with ld_classic" OFF)
 
 # Extension options
 option(CSP_BUILD_KAFKA_ADAPTER "Build kafka adapter" ON)
+option(CSP_BUILD_ARROW_ADAPTER "Build arrow adapter" ON)
 option(CSP_BUILD_PARQUET_ADAPTER "Build parquet adapter" ON)
 option(CSP_BUILD_WS_CLIENT_ADAPTER "Build ws client adapter" ON)
 
@@ -237,6 +238,10 @@ find_package(csp_autogen REQUIRED)
 find_package(DepsBase REQUIRED)
 
 # Adapter dependencies
+if(CSP_BUILD_ARROW_ADAPTER)
+    find_package(DepsArrowAdapter REQUIRED)
+endif()
+
 if(CSP_BUILD_KAFKA_ADAPTER)
     find_package(DepsKafkaAdapter REQUIRED)
 endif()

Makefile

@@ -194,16 +194,17 @@ dependencies-mac:  ## install dependencies for mac
 	brew unlink bison flex && brew link --force bison flex
 
 dependencies-debian:  ## install dependencies for linux
-	apt-get install -y autoconf autoconf-archive automake bison cmake curl flex libtool ninja-build pkg-config tar unzip zip
+	apt-get install -y autoconf autoconf-archive automake bison cmake curl flex libtool ninja-build pkg-config tar unzip
 
 dependencies-fedora:  ## install dependencies for linux
-	yum install -y autoconf autoconf-archive automake bison ccache cmake curl flex libtool perl-IPC-Cmd pkg-config tar unzip zip
+	yum install -y autoconf autoconf-archive automake bison ccache cmake curl flex libtool perl-IPC-Cmd pkg-config tar unzip
 
 dependencies-vcpkg:  ## install dependencies via vcpkg
 	cd vcpkg && ./bootstrap-vcpkg.sh && ./vcpkg install
 
 dependencies-win:  ## install dependencies via windows
-	choco install cmake curl winflexbison ninja unzip zip --no-progress -y
+	choco install cmake --version=3.31.6 --allow-downgrade
+	choco install curl winflexbison ninja unzip --no-progress -y
 
 ############################################################################################
 # Thanks to Francoise at marmelab.com for this

conda/dev-environment-unix.yml

@@ -3,6 +3,7 @@ channels:
   - conda-forge
   - nodefaults
 dependencies:
+ - astor
  - bison
  - brotli
  - bump-my-version
@@ -16,7 +17,7 @@ dependencies:
  - graphviz
  - gtest
  - httpx>=0.20,<1
- - libarrow=16
+ - libarrow<20
  - libboost>=1.80.0
  - libboost-headers>=1.80.0
  - librdkafka
@@ -27,11 +28,12 @@ dependencies:
  - mdformat-tables>=1,<1.1
  - ninja
  - numpy<2
- - pandas
+ - pandas<2.3  # [py<310]
+ - pandas      # [py>=310]
  - pillow
  - polars
  - psutil
- - pyarrow>=15,<19
+ - pyarrow>=15,<20
  - pydantic>=2
  - pytest
  - pytest-asyncio

conda/dev-environment-win.yml

@@ -4,6 +4,7 @@ channels:
   - nodefaults
 dependencies:
 #  - bison  # not available on windows
+ - astor
  - brotli
  - bump-my-version
  - cmake
@@ -16,7 +17,7 @@ dependencies:
  - graphviz
  - gtest
  - httpx>=0.20,<1
- - libarrow=16
+ - libarrow<20
  - libboost>=1.80.0
  - libboost-headers>=1.80.0
  - librdkafka
@@ -27,11 +28,12 @@ dependencies:
  - mdformat-tables>=1,<1.1
  - ninja
  - numpy<2
- - pandas
+ - pandas<2.3  # [py<310]
+ - pandas      # [py>=310]
  - pillow
  - polars
  - psutil
- - pyarrow>=7,<19
+ - pyarrow>=15,<20
  - pydantic>=2
  - pytest
  - pytest-asyncio

cpp/cmake/modules/FindDepsArrowAdapter.cmake

@@ -0,0 +1,5 @@
+cmake_minimum_required(VERSION 3.7.2)
+
+# ARROW
+find_package(Arrow REQUIRED)
+include_directories(${ARROW_INCLUDE_DIR})

cpp/csp/adapters/parquet/ParquetFileWriterWrapper.cpp

@@ -21,10 +21,9 @@ void ParquetFileWriterWrapper::openImpl( const std::string &fileName, const std:
     ::parquet::ArrowWriterProperties::Builder arrowBuilder;
     arrowBuilder.store_schema();
 
-    STATUS_OK_OR_THROW_RUNTIME(
-        ::parquet::arrow::FileWriter::Open( *getSchema(), arrow::default_memory_pool(), m_outputStream, builder.build(), arrowBuilder.build(),
-                                            &m_fileWriter ),
-            "Failed to open parquet file writer" );
+    auto res = ::parquet::arrow::FileWriter::Open( *getSchema(), arrow::default_memory_pool(), m_outputStream, builder.build(), arrowBuilder.build() );
+    STATUS_OK_OR_THROW_RUNTIME( res.status(), "Failed to open parquet file writer" );
+    m_fileWriter = res.MoveValueUnsafe();
 }
 
 void ParquetFileWriterWrapper::close()

cpp/csp/adapters/parquet/ParquetReader.h

@@ -221,28 +221,7 @@ class ParquetReader
     {
         if( symbol )
             validateSymbolType( symbol.value() );
-
-        auto columnAdapterReference = (*this)[column];
-
-        if(columnAdapterReference->getContainerValueType()->type() == csp::CspType::Type::STRING)
-        {
-            auto &listAdapter = dynamic_cast<ListColumnAdapter<arrow::StringArray, std::string> &>(*columnAdapterReference);
-            listAdapter.addSubscriber( inputAdapter, symbol,
-                                         listReaderInterface );
-        }
-        else
-        {
-            PartialSwitchCspType<csp::CspType::Type::BOOL, csp::CspType::Type::INT64, csp::CspType::Type::DOUBLE>::invoke(
-                    columnAdapterReference -> getContainerValueType().get(),
-                    [ &columnAdapterReference, &listReaderInterface, &symbol, &inputAdapter ]( auto tag )
-                    {
-                        using T = typename decltype(tag)::type;
-                        using ArrowArrayType = typename arrow::TypeTraits<typename arrow::CTypeTraits<T>::ArrowType>::ArrayType;
-                        auto &listAdapter = dynamic_cast<ListColumnAdapter<ArrowArrayType> &>(*columnAdapterReference);
-                        listAdapter.addSubscriber( inputAdapter, symbol,
-                                                     listReaderInterface );
-                    } );
-        }
+        (*this)[column]->addSubscriber( inputAdapter, symbol, listReaderInterface );
     }
 
     /**