diff --git a/.claude/docs-guidelines/repo-conventions.md b/.claude/docs-guidelines/repo-conventions.md index 86dff5144..2cc294074 100644 --- a/.claude/docs-guidelines/repo-conventions.md +++ b/.claude/docs-guidelines/repo-conventions.md @@ -121,7 +121,10 @@ Do not write `index.html` stubs under `docs/`. - No absolute links to `https://documentation.eccenca.com/latest/...` for internal pages. - No base-relative links such as `/automate/...`. - Same-page references use a plain anchor: `[Configure OAuth clients](#configure-oauth-clients-helm)`, never a full path back to the same file. -- `task check:links` runs the link checker; `task build` runs `zensical build --strict` and fails on unresolved internal links. +- `task check:links` runs [lychee](https://lychee.cli.rs/) over the outgoing http(s) links; exclusions and accepted + status codes live in `lychee.toml`. `task build` runs `zensical build --strict` and fails on unresolved internal links. +- An IRI written as an autolink (``) is checked like any link. Identifier namespaces + that do not resolve are excluded in `lychee.toml`; add a new one there rather than formatting the IRI differently. ## Images diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 150417d9d..bb352a132 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -70,7 +70,28 @@ jobs: - name: install packages run: sudo apt-get install -y libffi-dev libz-dev + - name: install lychee + env: + LYCHEE_VERSION: "0.24.2" + LYCHEE_ASSET: lychee-x86_64-unknown-linux-gnu + LYCHEE_RELEASES: https://github.com/lycheeverse/lychee/releases/download + run: | + curl -sSfL "${LYCHEE_RELEASES}/lychee-v${LYCHEE_VERSION}/${LYCHEE_ASSET}.tar.gz" \ + | sudo tar -xz -C /usr/local/bin --strip-components=1 "${LYCHEE_ASSET}/lychee" + lychee --version + + - name: load link check cache + uses: actions/cache@v4 + with: + path: .lycheecache + # A new key per run saves the updated cache; restore-keys picks up the latest one. + # lychee itself re-checks entries older than max_cache_age (lychee.toml). + key: lychee-${{ github.run_id }} + restore-keys: lychee- + - name: check + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: task check - name: build diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3ab2b9aad..70e1728c6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -61,7 +61,28 @@ jobs: - name: test run: task test:unit + - name: install lychee + env: + LYCHEE_VERSION: "0.24.2" + LYCHEE_ASSET: lychee-x86_64-unknown-linux-gnu + LYCHEE_RELEASES: https://github.com/lycheeverse/lychee/releases/download + run: | + curl -sSfL "${LYCHEE_RELEASES}/lychee-v${LYCHEE_VERSION}/${LYCHEE_ASSET}.tar.gz" \ + | sudo tar -xz -C /usr/local/bin --strip-components=1 "${LYCHEE_ASSET}/lychee" + lychee --version + + - name: load link check cache + uses: actions/cache@v4 + with: + path: .lycheecache + # A new key per run saves the updated cache; restore-keys picks up the latest one. + # lychee itself re-checks entries older than max_cache_age (lychee.toml). + key: lychee-${{ github.run_id }} + restore-keys: lychee- + - name: check + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: task check - name: Publish Test Results diff --git a/.gitignore b/.gitignore index 60a4a35a2..9e94bf203 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ tmp .python-version dist/ .env +.lycheecache *.sw[a-p] *~ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1a0dfddb5..f09fa4aa2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -33,6 +33,7 @@ The following tools you need locally to get started: - [poetry](https://python-poetry.org/) - [task](https://taskfile.dev/) +- [lychee](https://lychee.cli.rs/) - the link checker behind `task check:links` - git, markdown editor On a few OS distributions (e.g. Arch Linux) the tool/binary is named `go-task`. @@ -114,7 +115,7 @@ On this page is search function for icons available as well. |Success|Success admonitions are a type of documentation element used to highlight successful outcomes or positive results associated with a particular task, process, or feature|Graph is created successfully.| |Bug|A bug admonition include a description of the bug or issue, as well as steps that the user can take to avoid or work around the problem. It may also include information about when the bug will be fixed or patched, if applicable.|Users may experience issues with the file saving feature when running this software on Windows 10. To avoid data loss or corruption, be sure to save your work frequently and consider using an external backup device. Our development team is working to resolve this issue in the next software update.| |Example|The example admonition is typically used in instructional or educational documents to clarify complex concepts or demonstrate how to perform a specific task.|To create a new email account, click on the "Sign Up" button on the homepage and enter your name, email address, and desired password. Be sure to choose a strong password with a mix of uppercase and lowercase letters, numbers, and special characters. Once you have entered your information, click "Create Account" to complete the process.| -|Task|The Task admonition type is used to describe a series of steps or tasks required to complete a process or action. Clear and concise descriptions of each step, along with any relevant details or instructions, should be included. It's helpful to use images or diagrams to illustrate the process. Overall, The Task admonition is useful for providing detailed instructions to readers.|Loading Data into a CMEM involves 3 steps:
1. Connect to the database.
2. Choose the data file and appropriate file type.
3. Run workflow to import.| +|Task|The Task admonition type is used to describe a series of steps or tasks required to complete a process or action. Clear and concise descriptions of each step, along with any relevant details or instructions, should be included. It's helpful to use images or diagrams to illustrate the process. Overall, The Task admonition is useful for providing detailed instructions to readers.|Loading Data into eccenca Corporate Memory involves 3 steps:
1. Connect to the database.
2. Choose the data file and appropriate file type.
3. Run workflow to import.| diff --git a/Taskfile.yml b/Taskfile.yml index 38d8b4c6d..4f7f30ebb 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -118,11 +118,17 @@ tasks: - poetry run dec-tool check-zensical-output check:links: - desc: Check outgoing links - deps: - - install + desc: Check outgoing links (lychee, configured in lychee.toml) + summary: | + Checks the outgoing http(s) links of the Markdown sources under docs/ + with lychee. Exclusions, accepted status codes and retries are set in + lychee.toml. Set GITHUB_TOKEN to avoid GitHub rate limits. + Shows a progress bar in a terminal; in CI (CI is set) it prints the summary only. + preconditions: + - sh: command -v lychee + msg: "lychee is not installed - see https://lychee.cli.rs/installation/ (e.g. brew install lychee)" cmds: - - poetry run linkcheckMarkdown -r docs 2>&1 | grep -v "ResourceWarning" + - lychee --config lychee.toml ${CI:+--no-progress} 'docs/**/*.md' test: desc: Run the Python unit test suite diff --git a/data/integrations.yml b/data/integrations.yml index 3702bbd47..705b100b7 100644 --- a/data/integrations.yml +++ b/data/integrations.yml @@ -163,7 +163,7 @@ integrations: - name: GraphQL icon: ":simple-graphql:" content: | - Execute a {{p.cmem_plugin_graphql_workflow_graphql_GraphQLPlugin}} and process the result in a workflow. + Execute a {{p.cmem_plugin_graphql_Query}} and process the result in a workflow. - name: eMail / SMTP / IMAP icon: ":material-email-outline:" @@ -254,7 +254,7 @@ integrations: - name: eccenca Marketplace icon: ":eccenca-module-marketplace:" content: | - Manage [Marketplace Packages](../../develop/packages/index.md) in workflows: + Manage [Marketplace Packages](../../develop/marketplace-packages/index.md) in workflows: [build package archives]({{p.cmem_plugin_packages_BuildPackages_ref}}), [bump their version]({{p.cmem_plugin_packages_BumpVersions_ref}}), [publish]({{p.cmem_plugin_packages_PublishPackages_ref}}) them to a Marketplace Server diff --git a/data/plugins.json b/data/plugins.json index f341463ea..02408405c 100644 --- a/data/plugins.json +++ b/data/plugins.json @@ -435,14 +435,14 @@ "Uncategorized" ], "main_category": "Uncategorized", - "description": "Fetch and output LLM created embeddings from input entities.", - "markdownDocumentation": "\nThis plugin creates vector embeddings from text data using an OpenAI compatible embeddings API.\nIt processes input entities containing text data and generates high-dimensional vector\nrepresentations that capture semantic meaning.\n\n## Features\n\n- Supports OpenAI embeddings models (e.g., `text-embedding-3-small`)\n- Batch processing for efficient API usage\n- Configurable input/output paths\n- Built-in error handling and workflow cancellation support\n\n## Input/Output\n\n- **Input**: Entities with text data in specified paths\n- **Output**: Original entities enhanced with embedding vectors and source text\n- Embedding vectors are stored as string representations of float arrays\n- Source text used for embedding is preserved for reference\n\n## Use Cases\n\n- Semantic search and similarity matching\n- Text clustering and classification\n- Recommendation systems\n- Natural language processing pipelines", + "description": "Turn the text of each entity into a vector embedding and add it to the entity.", + "markdownDocumentation": "\nTurns the text an entity carries into a vector embedding and adds it to that entity, using an\nembeddings model reached over the OpenAI embeddings protocol. Embeddings are what makes\nsimilarity search possible: two entities whose text means roughly the same thing get vectors\nthat sit close to one another, which is the basis for semantic search, clustering and\nretrieval-augmented generation.\n\n## Ports\n\nEntities arrive on one input port and leave on one output port with two paths added - the\nvector, and the text the vector was computed from. Keeping the source text means a later\nreader can see exactly what was embedded rather than having to reconstruct it.\n\nNaming the paths to embed fixes the shape of both ports: the input port asks for those paths\nand for any paths you forward, and the output port carries all of them plus the two new ones.\nLeaving the selection empty embeds everything an entity carries, which no schema can describe\nin advance, so the input port becomes flexible and the output schema is settled only at run\ntime.\n\nBoth added values are text. The vector is the text form of a Python list of floats, such as\n`[0.021, -0.118, ...]`, not a JSON array. The source text is the value itself when the entity\nholds a single value, or when a single selected path holds a single value; otherwise it is the\ntext form of a Python dictionary from path to values - again not JSON, so single quotes rather\nthan double ones.\n\n## Where it fits\n\nEmbeddings are of no use on their own: they are written on to a store that can search them,\nand the usual chain is a dataset or a knowledge graph, then this task, then that store. Its\nsibling **Execute Instructions** is the generating half of the same picture, so the two often\nappear in one workflow - this task finding the entities worth looking at, and that one writing\nabout them.\n\n## Caveats\n\nEntities are sent in chunks and a whole chunk is embedded in one request. Nothing catches a\nfailure of that request, so a rate limit, a text longer than the model's context window, or an\nentity the provider rejects stops the workflow and takes the rest of the chunk with it.\nCancelling likewise takes effect between chunks.\n\nAn output path may not be a path the incoming entities already carry. This is checked when the\ntask is configured and again against the schema that actually arrives, and is an error either\nway.\n\nEmbeddings are comparable only when they come from the same model. Changing the model means\neverything already stored has to be computed again.\n", "pluginIcon": "data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiIHN0YW5kYWxvbmU9Im5vIj8+CjwhLS0gVXBsb2FkZWQgdG86IFNWRyBSZXBvLCB3d3cuc3ZncmVwby5jb20sIEdlbmVyYXRvcjogU1ZHIFJlcG8gTWl4ZXIgVG9vbHMgLS0+Cgo8c3ZnCiAgIHdpZHRoPSI4MDBweCIKICAgaGVpZ2h0PSI4MDBweCIKICAgdmlld0JveD0iMCAwIDEwMDAgMTAwMCIKICAgaWQ9IkxheWVyXzIiCiAgIHZlcnNpb249IjEuMSIKICAgeG1sOnNwYWNlPSJwcmVzZXJ2ZSIKICAgc29kaXBvZGk6ZG9jbmFtZT0iY3JlYXRlX2VtYmVkZGluZy5zdmciCiAgIGlua3NjYXBlOnZlcnNpb249IjEuMS4yIChiOGUyNWJlOCwgMjAyMi0wMi0wNSkiCiAgIHhtbG5zOmlua3NjYXBlPSJodHRwOi8vd3d3Lmlua3NjYXBlLm9yZy9uYW1lc3BhY2VzL2lua3NjYXBlIgogICB4bWxuczpzb2RpcG9kaT0iaHR0cDovL3NvZGlwb2RpLnNvdXJjZWZvcmdlLm5ldC9EVEQvc29kaXBvZGktMC5kdGQiCiAgIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIKICAgeG1sbnM6c3ZnPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+PGRlZnMKICAgICBpZD0iZGVmczgyOSIgLz48c29kaXBvZGk6bmFtZWR2aWV3CiAgICAgaWQ9Im5hbWVkdmlldzgyNyIKICAgICBwYWdlY29sb3I9IiNmZmZmZmYiCiAgICAgYm9yZGVyY29sb3I9IiM2NjY2NjYiCiAgICAgYm9yZGVyb3BhY2l0eT0iMS4wIgogICAgIGlua3NjYXBlOnBhZ2VzaGFkb3c9IjIiCiAgICAgaW5rc2NhcGU6cGFnZW9wYWNpdHk9IjAuMCIKICAgICBpbmtzY2FwZTpwYWdlY2hlY2tlcmJvYXJkPSIwIgogICAgIHNob3dncmlkPSJmYWxzZSIKICAgICBpbmtzY2FwZTp6b29tPSIxLjA1ODc1IgogICAgIGlua3NjYXBlOmN4PSI0MDAiCiAgICAgaW5rc2NhcGU6Y3k9IjQwMCIKICAgICBpbmtzY2FwZTp3aW5kb3ctd2lkdGg9IjE5MjAiCiAgICAgaW5rc2NhcGU6d2luZG93LWhlaWdodD0iMTAyNyIKICAgICBpbmtzY2FwZTp3aW5kb3cteD0iMTcyOCIKICAgICBpbmtzY2FwZTp3aW5kb3cteT0iMjUiCiAgICAgaW5rc2NhcGU6d2luZG93LW1heGltaXplZD0iMSIKICAgICBpbmtzY2FwZTpjdXJyZW50LWxheWVyPSJMYXllcl8yIiAvPjxwYXRoCiAgICAgZD0ibSA4NTguMTkwNzMsNDYwLjc2NDQ5IGMgLTE2LjYyNjQ2LDAgLTMwLjgyNzkyLDEwLjM0Nzk3IC0zNi41NDE4OCwyNC45NTA1NSBoIC03Mi45MTU3NyB2IC02OS4xMzQyNyBoIDM5Ljk0MjgzIGMgNy44ODg2OCwwIDE0LjI4NDg4LC02LjM5NTA2IDE0LjI4NDg4LC0xNC4yODQ4OSAwLC03Ljg4OTgzIC02LjM5NjIsLTE0LjI4NDg5IC0xNC4yODQ4OCwtMTQuMjg0ODkgaCAtMzkuOTQyODMgdiAtNjkuMTM0MjcgaCAzOS45NDI4MyBjIDcuODg4NjgsMCAxNC4yODQ4OCwtNi4zOTUwNiAxNC4yODQ4OCwtMTQuMjg0ODkgMCwtNy44ODg2OSAtNi4zOTYyLC0xNC4yODQ4OSAtMTQuMjg0ODgsLTE0LjI4NDg5IGggLTM5Ljk0MjgzIHYgLTQuMzQ5NDYgYyAwLC0xOC45MzQ5IC0xNS4zNDg4MywtMzQuMjgzNzMgLTM0LjI4MzczLC0zNC4yODM3MyBoIC00LjkyMDg2IHYgLTQwLjUxNDIyIGMgMCwtNy44ODk4MyAtNi4zOTUwNSwtMTQuMjg0ODggLTE0LjI4NDg4LC0xNC4yODQ4OCAtNy44ODk4MywwIC0xNC4yODQ4OSw2LjM5NTA1IC0xNC4yODQ4OSwxNC4yODQ4OCB2IDQwLjUxNDIyIGggLTY5LjEzNDI3IHYgLTcyLjc1MTIxIGMgMTQuNjAyNTgsLTUuNzEzOTUgMjQuOTUxNjksLTE5LjkxNDI3IDI0Ljk1MTY5LC0zNi41NDE4OCAwLC0yMS42Njk2IC0xNy41NjY5OCwtMzkuMjM1NDQgLTM5LjIzNjU4LC0zOS4yMzU0NCAtMjEuNjY5NiwwIC0zOS4yMzU0NCwxNy41NjY5OCAtMzkuMjM1NDQsMzkuMjM1NDQgMCwxNi42Mjc2MSAxMC4zNDc5NywzMC44MjkwNyAyNC45NTA1NSwzNi41NDE4OCB2IDcyLjc1MTIxIEggNTE0LjEyMDQgdiAtNDAuNTE0MjIgYyAwLC03Ljg4OTgzIC02LjM5NjIxLC0xNC4yODQ4OCAtMTQuMjg0ODksLTE0LjI4NDg4IC03Ljg4OTgzLDAgLTE0LjI4NDg5LDYuMzk1MDUgLTE0LjI4NDg5LDE0LjI4NDg4IHYgNDAuNTE0MjIgaCAtNjkuMTM0MjcgdiAtNzIuNzUxMjEgYyAxNC42MDI1OCwtNS43MTM5NSAyNC45NTA1NSwtMTkuOTE0MjcgMjQuOTUwNTUsLTM2LjU0MTg4IDAsLTIxLjY2OTYgLTE3LjU2Njk4LC0zOS4yMzU0NCAtMzkuMjM1NDQsLTM5LjIzNTQ0IC0yMS42Njk2LDAgLTM5LjIzNTQ0LDE3LjU2Njk4IC0zOS4yMzU0NCwzOS4yMzU0NCAwLDE2LjYyNzYxIDEwLjM0Nzk3LDMwLjgyOTA3IDI0Ljk1MDU1LDM2LjU0MTg4IHYgNzIuNzUxMjEgSCAzMTguNzEyMyB2IC00MC41MTQyMiBjIDAsLTcuODg5ODMgLTYuMzk1MDYsLTE0LjI4NDg4IC0xNC4yODQ4OSwtMTQuMjg0ODggLTcuODg5ODMsMCAtMTQuMjg0ODksNi4zOTUwNSAtMTQuMjg0ODksMTQuMjg0ODggdiA0MC41MTQyMiBoIC00LjkyMDg1IGMgLTE4LjkzNDkxLDAgLTM0LjI4MzczLDE1LjM0ODgzIC0zNC4yODM3MywzNC4yODM3MyB2IDQuMzQ5NDYgaCAtMzkuOTQyODMgYyAtNy44ODk4MywwIC0xNC4yODQ4OCw2LjM5NjIgLTE0LjI4NDg4LDE0LjI4NDg5IDAsNy44ODk4MyA2LjM5NTA1LDE0LjI4NDg5IDE0LjI4NDg4LDE0LjI4NDg5IGggMzkuOTQyODMgdiA2OS4xMzQyNyBoIC0zOS45NDI4MyBjIC03Ljg4OTgzLDAgLTE0LjI4NDg4LDYuMzk1MDYgLTE0LjI4NDg4LDE0LjI4NDg5IDAsNy44ODk4MyA2LjM5NTA1LDE0LjI4NDg5IDE0LjI4NDg4LDE0LjI4NDg5IGggMzkuOTQyODMgdiA2OS4xMzQyNyBoIC03Mi41ODc3OSBjIC01LjcxMjgxLC0xNC42MDI1OCAtMTkuOTE0MjgsLTI0Ljk1MDU1IC0zNi41NDE4OCwtMjQuOTUwNTUgLTIxLjY2OTYsMCAtMzkuMjM1NDQsMTcuNTY2OTggLTM5LjIzNTQ0LDM5LjIzNTQ0IDAsMjEuNjY5NiAxNy41NjY5OCwzOS4yMzU0NCAzOS4yMzU0NCwzOS4yMzU0NCAxNi42MjY0NiwwIDMwLjgyOTA3LC0xMC4zNDc5NyAzNi41NDE4OCwtMjQuOTUwNTYgaCA3Mi41ODc3OSB2IDY5LjEzNDI4IGggLTM5Ljk0MjgzIGMgLTcuODg5ODMsMCAtMTQuMjg0ODgsNi4zOTUwNiAtMTQuMjg0ODgsMTQuMjg0ODkgMCw3Ljg4OTgzIDYuMzk1MDUsMTQuMjg0ODggMTQuMjg0ODgsMTQuMjg0ODggaCAzOS45NDI4MyB2IDY5LjEzNDI4IGggLTM5Ljk0MjgzIGMgLTcuODg5ODMsMCAtMTQuMjg0ODgsNi4zOTYyIC0xNC4yODQ4OCwxNC4yODQ4OSB2IDAgYyAwLDcuODg5ODMgNi4zOTUwNSwxNC4yODQ4OCAxNC4yODQ4OCwxNC4yODQ4OCBoIDM5Ljk0MjgzIHYgNS40OTM0IGMgMCwxOC45MzQ5IDE1LjM0ODgyLDM0LjI4MzczIDM0LjI4MzczLDM0LjI4MzczIGggNC45MjA4NSB2IDM5LjM3MTQzIGMgMCw3Ljg4ODY4IDYuMzk1MDYsMTQuMjg0ODggMTQuMjg0ODksMTQuMjg0ODggNy44ODk4MywwIDE0LjI4NDg5LC02LjM5NjIgMTQuMjg0ODksLTE0LjI4NDg4IHYgLTM5LjM3MTQzIGggNjkuMTM0MjcgdiA3MS42MDg0MiBjIC0xNC42MDI1OCw1LjcxMjgxIC0yNC45NTA1NSwxOS45MTQyNyAtMjQuOTUwNTUsMzYuNTQxODggMCwyMS42Njk2IDE3LjU2NTg0LDM5LjIzNTQ0IDM5LjIzNTQ0LDM5LjIzNTQ0IDIxLjY2OTYsMCAzOS4yMzU0NCwtMTcuNTY2OTggMzkuMjM1NDQsLTM5LjIzNTQ0IDAsLTE2LjYyNzYxIC0xMC4zNDc5NywtMzAuODI5MDcgLTI0Ljk1MDU1LC0zNi41NDE4OCB2IC03MS42MDg0MiBoIDY5LjEzNDI3IHYgMzkuMzcxNDMgYyAwLDcuODg4NjggNi4zOTUwNiwxNC4yODQ4OCAxNC4yODQ4OSwxNC4yODQ4OCA3Ljg4OTgzLDAgMTQuMjg0ODksLTYuMzk2MiAxNC4yODQ4OSwtMTQuMjg0ODggdiAtMzkuMzcxNDMgaCA2OS4xMzQyNyB2IDcxLjYwODQyIGMgLTE0LjYwMjU4LDUuNzEyODEgLTI0Ljk1MDU1LDE5LjkxNDI3IC0yNC45NTA1NSwzNi41NDE4OCAwLDIxLjY2OTYgMTcuNTY2OTgsMzkuMjM1NDQgMzkuMjM1NDQsMzkuMjM1NDQgMjEuNjY5NiwwIDM5LjIzNjU4LC0xNy41NjY5OCAzOS4yMzY1OCwtMzkuMjM1NDQgMCwtMTYuNjI3NjEgLTEwLjM0Nzk3LC0zMC44MjkwNyAtMjQuOTUxNjksLTM2LjU0MTg4IHYgLTcxLjYwODQyIGggNjkuMTM0MjcgdiAzOS4zNzE0MyBjIDAsNy44ODg2OCA2LjM5NTA2LDE0LjI4NDg4IDE0LjI4NDg5LDE0LjI4NDg4IDcuODg5ODMsMCAxNC4yODQ4OCwtNi4zOTYyIDE0LjI4NDg4LC0xNC4yODQ4OCB2IC0zOS4zNzE0MyBoIDQuOTIwODYgYyAxOC45MzQ5LDAgMzQuMjgzNzMsLTE1LjM0ODgzIDM0LjI4MzczLC0zNC4yODM3MyB2IC01LjQ5MzQgaCAzOS45NDI4MyBjIDcuODg4NjgsMCAxNC4yODQ4OCwtNi4zOTUwNSAxNC4yODQ4OCwtMTQuMjg0ODggdiAwIGMgMCwtNy44ODg2OSAtNi4zOTYyLC0xNC4yODQ4OSAtMTQuMjg0ODgsLTE0LjI4NDg5IGggLTM5Ljk0MjgzIHYgLTY5LjEzNDI4IGggMzkuOTQyODMgYyA3Ljg4ODY4LDAgMTQuMjg0ODgsLTYuMzk1MDUgMTQuMjg0ODgsLTE0LjI4NDg4IDAsLTcuODg5ODMgLTYuMzk2MiwtMTQuMjg0ODkgLTE0LjI4NDg4LC0xNC4yODQ4OSBoIC0zOS45NDI4MyB2IC02OS4xMzQyOCBoIDcyLjkxNTc3IGMgNS43MTM5NiwxNC42MDI1OSAxOS45MTQyOCwyNC45NTA1NiAzNi41NDE4OCwyNC45NTA1NiAyMS42Njk2LDAgMzkuMjM1NDQsLTE3LjU2Njk4IDM5LjIzNTQ0LC0zOS4yMzU0NCAwLC0yMS42Njk2IC0xNy41NjU4NCwtMzkuMjM1NDQgLTM5LjIzNTQ0LC0zOS4yMzU0NCB6IE0gNDc1Ljg1ODYxLDMyNC42NzUyNCBoIDE1My4wOTA1NiBjIDcuODg4NjgsMCAxNC4yODQ4OCw2LjM5NTA1IDE0LjI4NDg4LDE0LjI4NDg4IDAsNy44ODk4MyAtNi4zOTYyLDE0LjI4NDg5IC0xNC4yODQ4OCwxNC4yODQ4OSBIIDQ3NS44NTg2MSBjIC03Ljg4OTgyLDAgLTE0LjI4NDg4LC02LjM5NTA2IC0xNC4yODQ4OCwtMTQuMjg0ODkgMCwtNy44ODk4MyA2LjM5NjIsLTE0LjI4NDg4IDE0LjI4NDg4LC0xNC4yODQ4OCB6IG0gOTIuNDYzMjEsMTM4LjM1NDI2IGMgLTUuNTc5MSwtNS41NzkxIC01LjU3OTEsLTE0LjYyMzE1IDAsLTIwLjIwMjI2IDUuNTc5MTEsLTUuNTc5MSAxNC42MjMxNiwtNS41NzkxIDIwLjIwMTEyLDAgbCA0OC4yMTU0OSw0OC4yMTQzNSBjIDUuNTc3OTYsNS41NzkxMSA1LjU3Nzk2LDE0LjYyMzE1IDAsMjAuMjAyMjYgbCAtNDguMjE1NDksNDguMjE1NDkgYyAtNS41Nzc5Niw1LjU3Nzk2IC0xNC42MjMxNSw1LjU3Nzk2IC0yMC4yMDExMiwwIC01LjU3OTEsLTUuNTc5MTEgLTUuNTc5MSwtMTQuNjIzMTUgMCwtMjAuMjAyMjYgbCAzOC4xMTQzNywtMzguMTE0MzYgeiBNIDM3MC43MjE4NSwzMjQuNjc1MjQgaCA0Ny4xOTI2OSBjIDcuODg5ODMsMCAxNC4yODQ4OSw2LjM5NTA1IDE0LjI4NDg5LDE0LjI4NDg4IDAsNy44ODk4MyAtNi4zOTUwNiwxNC4yODQ4OSAtMTQuMjg0ODksMTQuMjg0ODkgaCAtNDcuMTkyNjkgYyAtNy44ODk4MywwIC0xNC4yODQ4OCwtNi4zOTUwNiAtMTQuMjg0ODgsLTE0LjI4NDg5IDAsLTcuODg5ODMgNi4zOTYyLC0xNC4yODQ4OCAxNC4yODQ4OCwtMTQuMjg0ODggeiBtIC02OS4zNTAyNiwtMzUuNDQ5MzggYyA3Ljg4OTgzLDAgMTQuMjg0ODgsNi4zOTYyIDE0LjI4NDg4LDE0LjI4NDg5IDAsNy44ODk4MyAtNi4zOTUwNSwxNC4yODQ4OSAtMTQuMjg0ODgsMTQuMjg0ODkgLTcuODg5ODMsMCAtMTQuMjg0ODksLTYuMzk1MDYgLTE0LjI4NDg5LC0xNC4yODQ4OSAwLC03Ljg4ODY5IDYuMzk1MDYsLTE0LjI4NDg5IDE0LjI4NDg5LC0xNC4yODQ4OSB6IG0gMCwzOTQuMTIxMTUgYyA3Ljg4OTgzLDAgMTQuMjg0ODgsNi4zOTYyIDE0LjI4NDg4LDE0LjI4NDg5IDAsNy44ODk4MyAtNi4zOTUwNSwxNC4yODQ4OCAtMTQuMjg0ODgsMTQuMjg0ODggLTcuODg5ODMsMCAtMTQuMjg0ODksLTYuMzk1MDUgLTE0LjI4NDg5LC0xNC4yODQ4OCAwLC03Ljg4ODY5IDYuMzk1MDYsLTE0LjI4NDg5IDE0LjI4NDg5LC0xNC4yODQ4OSB6IE0gMzQyLjI3MDkzLDU2My42NDMxIGggLTMxLjQxMDc1IGMgLTcuODg5ODMsMCAtMTQuMjg0ODksLTYuMzk1MDYgLTE0LjI4NDg5LC0xNC4yODQ4OSAwLC03Ljg4OTgzIDYuMzk1MDYsLTE0LjI4NDg5IDE0LjI4NDg5LC0xNC4yODQ4OSBoIDMxLjQxMDc1IGMgNy44ODk4MywwIDE0LjI4NDg5LDYuMzk1MDYgMTQuMjg0ODksMTQuMjg0ODkgMCw3Ljg4OTgzIC02LjM5NTA2LDE0LjI4NDg5IC0xNC4yODQ4OSwxNC4yODQ4OSB6IG0gNzUuNjQzNjEsMTEyLjgyNTQ1IGggLTQ3LjE5MjY5IGMgLTcuODg5ODMsMCAtMTQuMjg0ODgsLTYuMzk1MDUgLTE0LjI4NDg4LC0xNC4yODQ4OCAwLC03Ljg4OTgzIDYuMzk1MDUsLTE0LjI4NDg5IDE0LjI4NDg4LC0xNC4yODQ4OSBoIDQ3LjE5MjY5IGMgNy44ODk4MywwIDE0LjI4NDg5LDYuMzk1MDYgMTQuMjg0ODksMTQuMjg0ODkgMCw3Ljg4ODY4IC02LjM5NTA2LDE0LjI4NDg4IC0xNC4yODQ4OSwxNC4yODQ4OCB6IG0gMTMuNDM1OCwtMTM3LjIxMTQ3IGMgNS41NzkxLDUuNTc3OTYgNS41NzkxLDE0LjYyMzE1IDAsMjAuMjAyMjYgLTUuNTc5MTEsNS41Nzc5NiAtMTQuNjIzMTUsNS41Nzc5NiAtMjAuMjAxMTIsMCBsIC00OC4yMTU0OSwtNDguMjE1NDkgYyAtNS41Nzc5NiwtNS41Nzc5NiAtNS41Nzc5NiwtMTQuNjIzMTUgMCwtMjAuMjAyMjYgbCA0OC4yMTU0OSwtNDguMjE0MzUgYyA1LjU3Nzk3LC01LjU3OTEgMTQuNjIzMTUsLTUuNTc5MSAyMC4yMDExMiwwIDUuNTc5MSw1LjU3Nzk3IDUuNTc5MSwxNC42MjMxNiAwLDIwLjIwMjI2IGwgLTM4LjExMzIyLDM4LjExMzIyIHogbSAzMC45Mzg3NywzMC4xMDIyNiA0Ny45NjE4LC0xNDUuMzg0NzIgYyAyLjQ3MTg1LC03LjQ5MjEzIDEwLjU0OTEsLTExLjU2MTYxIDE4LjA0MTI0LC05LjA5MDkgNy40OTIxMywyLjQ3MTg2IDExLjU2MTYxLDEwLjU0OTEgOS4wOTA5LDE4LjA0MTI0IGwgLTQ3Ljk2MTc5LDE0NS4zODQ3MSBjIC0yLjQ3MTg2LDcuNDkyMTQgLTEwLjU0OTExLDExLjU2MTYyIC0xOC4wNDEyNCw5LjA5MDkxIC03LjQ5MjE0LC0yLjQ3MDcyIC0xMS41NjE2MiwtMTAuNTQ5MTEgLTkuMDkwOTEsLTE4LjA0MTI0IHogbSAxNjYuNjYxMiwxMDcuMTA5MjEgaCAtMTUzLjA5MTcgYyAtNy44ODk4MiwwIC0xNC4yODQ4OCwtNi4zOTUwNSAtMTQuMjg0ODgsLTE0LjI4NDg4IDAsLTcuODg5ODMgNi4zOTUwNiwtMTQuMjg0ODkgMTQuMjg0ODgsLTE0LjI4NDg5IGggMTUzLjA5MDU2IGMgNy44ODg2OCwwIDE0LjI4NDg4LDYuMzk1MDYgMTQuMjg0ODgsMTQuMjg0ODkgMTBlLTQsNy44ODg2OCAtNi4zOTUwNiwxNC4yODQ4OCAtMTQuMjgzNzQsMTQuMjg0ODggeiBtIDE0LjE2NjA0LC0xMjcuMTEwMzQgYyAwLC03Ljg4OTgzIDYuMzk1MDUsLTE0LjI4NDg5IDE0LjI4NDg4LC0xNC4yODQ4OSBoIDMxLjQxMDc1IGMgNy44ODk4MywwIDE0LjI4NDg5LDYuMzk1MDYgMTQuMjg0ODksMTQuMjg0ODkgMCw3Ljg4OTgzIC02LjM5NTA2LDE0LjI4NDg5IC0xNC4yODQ4OSwxNC4yODQ4OSBoIC0zMS40MTA3NSBjIC03Ljg4OTgzLDAgLTE0LjI4NDg4LC02LjM5NTA2IC0xNC4yODQ4OCwtMTQuMjg0ODkgeiBtIDU1LjE4NDIyLDE2Mi41NTg1NyBjIC03Ljg4OTgyLDAgLTE0LjI4NDg4LC02LjM5NTA1IC0xNC4yODQ4OCwtMTQuMjg0ODggMCwtNy44ODg2OSA2LjM5NTA2LC0xNC4yODQ4OSAxNC4yODQ4OCwtMTQuMjg0ODkgNy44ODk4MywwIDE0LjI4NDg5LDYuMzk2MiAxNC4yODQ4OSwxNC4yODQ4OSAwLDcuODg5ODMgLTYuMzk1MDYsMTQuMjg0ODggLTE0LjI4NDg5LDE0LjI4NDg4IHogbSAwLC0zOTQuMTIxMTQgYyAtNy44ODk4MiwwIC0xNC4yODQ4OCwtNi4zOTUwNiAtMTQuMjg0ODgsLTE0LjI4NDg5IDAsLTcuODg4NjkgNi4zOTUwNiwtMTQuMjg0ODkgMTQuMjg0ODgsLTE0LjI4NDg5IDcuODg5ODMsMCAxNC4yODQ4OSw2LjM5NjIgMTQuMjg0ODksMTQuMjg0ODkgMCw3Ljg4OTgzIC02LjM5NTA2LDE0LjI4NDg5IC0xNC4yODQ4OSwxNC4yODQ4OSB6IgogICAgIGlkPSJwYXRoODI0IgogICAgIHN0eWxlPSJzdHJva2Utd2lkdGg6MS4xNDI3OSIgLz48L3N2Zz4K", "properties": { "model": { "name": "model", "title": "Embeddings model", - "description": "The identifier of the embeddings model to use. Available model IDs for some public providers can be found here: [Claude](https://docs.claude.com/en/docs/build-with-claude/embeddings#available-models), [OpenAI](https://platform.openai.com/docs/guides/embeddings#embedding-models).", + "description": "The identifier of the embeddings model to use. Some providers do not offer a model listing, in which case type the identifier in directly rather than picking it from the list. Model identifiers for some public providers: [Claude](https://docs.claude.com/en/docs/build-with-claude/embeddings#available-models), [OpenAI](https://platform.openai.com/docs/guides/embeddings#embedding-models).", "type": "string", "parameterType": "string", "value": "", @@ -453,7 +453,7 @@ "embedding_paths": { "name": "embedding_paths", "title": "Embedding entity paths (comma-separated list)", - "description": "Changing this value will change, which input paths are used by the workflow task to calculate embeddings. A blank value means, all paths are used.", + "description": "The entity paths whose values are embedded, separated by commas. Naming paths here fixes the schema of both ports. Leaving it empty embeds every path an entity carries and leaves both ports open instead.", "type": "string", "parameterType": "string", "value": "text", @@ -464,7 +464,7 @@ "forward_paths": { "name": "forward_paths", "title": "Forward entity paths (comma-separated list)", - "description": "Paths from input entities to forward to output without modification. These paths will be passed through unchanged alongside embeddings.", + "description": "Further entity paths carried to the output unchanged, comma-separated. A path named here that an incoming entity does not have arrives empty rather than missing, so the output schema holds whatever was promised.", "type": "string", "parameterType": "string", "value": "", @@ -477,7 +477,7 @@ "base_url": { "name": "base_url", "title": "Base URL", - "description": "The base URL of the OpenAI compatible API (without endpoint path). If left empty CMEMs internal LLM proxy is used.", + "description": "The base URL of the OpenAI compatible API, without the endpoint path. Left empty, the task uses the built-in large language model proxy of the eccenca Corporate Memory deployment it runs in. That fallback does not apply to `AZURE_OPENAI`, which always needs a URL here.", "type": "string", "parameterType": "string", "value": "", @@ -487,8 +487,8 @@ }, "api_type": { "name": "api_type", - "title": "API Type", - "description": "Select the API client type. This determines the authentication method and endpoint configuration used for API requests. Choose `OPENAI` for direct OpenAI API access or `AZURE_OPENAI` for Azure-hosted OpenAI services. Consider using the API version advanced parameter in case you access Azure-hosted OpenAI services.", + "title": "API type", + "description": "Which client is used to reach the API. `OPENAI` speaks the plain OpenAI protocol and covers OpenAI itself as well as every compatible service. `AZURE_OPENAI` uses Azure's authentication and URL layout, and needs both a base URL and an API version.", "type": "string", "parameterType": "enumeration", "value": "OPENAI", @@ -499,7 +499,7 @@ "api_key": { "name": "api_key", "title": "API key", - "description": "An optional API key for authentication. When using CMEMs internal LLM proxy this parameter is ignored and `EXPLORE_AI_APIKEY` in `environments/config.env` is used.", + "description": "The key used to authenticate against the API. It is ignored when the built-in model proxy is used, which authenticates with the `EXPLORE_AI_APIKEY` entry of `environments/config.env` instead.", "type": "string", "parameterType": "password", "value": "", @@ -509,8 +509,8 @@ }, "api_version": { "name": "api_version", - "title": "API Version", - "description": "Azure OpenAI API version (only used when API Type is `AZURE_OPENAI`). For more information about OpenAI API version at Azure, please see [the documentation](https://learn.microsoft.com/en-gb/azure/ai-foundry/openai/api-version-lifecycle).", + "title": "API version", + "description": "The Azure OpenAI API version, read only when the API type is `AZURE_OPENAI`. The versions currently offered are listed in [the Azure documentation](https://learn.microsoft.com/en-gb/azure/ai-foundry/openai/api-version-lifecycle).", "type": "string", "parameterType": "string", "value": "", @@ -520,8 +520,8 @@ }, "timout_single_request": { "name": "timout_single_request", - "title": "Timeout (milliseconds)", - "description": "The timeout for a single API request in milliseconds.", + "title": "Timeout (seconds)", + "description": "How long a single API request may take, in seconds. One request covers a whole chunk of entities, so this limits embedding the chunk rather than embedding one entity.", "type": "string", "parameterType": "Long", "value": "10000", @@ -531,8 +531,8 @@ }, "entries_processing_buffer": { "name": "entries_processing_buffer", - "title": "Entries Processing Buffer", - "description": "How many input values do you want to send per request?", + "title": "Entries processing buffer", + "description": "How many entities are embedded per API request. Larger chunks mean fewer requests, but a chunk is also the unit that is lost when a request fails, and providers cap how much text one request may carry.", "type": "string", "parameterType": "Long", "value": "100", @@ -542,8 +542,8 @@ }, "embedding_output_text": { "name": "embedding_output_text", - "title": "Entity Embedding text (output)", - "description": "Changing this value will change the output schema accordingly. Default: _embedding_source", + "title": "Entity embedding text (output)", + "description": "The entity path the embedded source text is written to. It may not be a path the incoming entities already carry.", "type": "string", "parameterType": "string", "value": "_embedding_source", @@ -553,8 +553,8 @@ }, "embedding_output_path": { "name": "embedding_output_path", - "title": "Entity Embedding path (output)", - "description": "Changing this value will change the output schema accordingly. Default: _embedding", + "title": "Entity embedding path (output)", + "description": "The entity path the vector is written to. It may not be a path the incoming entities already carry.", "type": "string", "parameterType": "string", "value": "_embedding", @@ -1407,6 +1407,197 @@ "pluginType": "customtask", "relatedPlugins": [] }, + "cmem_plugin_s3-Download": { + "pluginId": "cmem_plugin_s3-Download", + "title": "Download S3 objects", + "categories": [ + "Uncategorized" + ], + "main_category": "Uncategorized", + "description": "Download objects from a bucket in S3 or S3-compatible object storage.", + "markdownDocumentation": "\nDownloads objects from a bucket into the workflow.\n\n## Ports\n\nObjects leave the task as file entities, written into a temporary directory of\nthe DataIntegration node. They are not added to the project, so anything that is\nmeant to be kept has to be written somewhere by a following task.\n\nThe task has two mutually exclusive shapes. With a bucket configured on it, it\nselects objects itself and takes no input. With the bucket left empty, the\nobjects to fetch arrive on an input port instead, in the form **List S3 objects**\nemits, and the settings that would narrow a selection down have nothing left to\nact on - the task in front has already decided.\n\n## Where the task sits\n\n**List S3 objects** produces the listing this task accepts, so a workflow can\nlist, narrow the result down and download only what is left. **Upload files to\nS3** covers the other direction, and the file entities this task emits are what\nevery file consuming task in the fleet accepts.\n\n## Caveats\n\n* **Each object is written into a numbered directory of its own**, so two keys\n ending in the same name do not overwrite each other. Only the name is kept,\n not the key's path, and a following task therefore sees the name rather than\n the key.\n* **An object replaced since it was listed is refused, not fetched.** The entity\n tag from the listing is sent with the request, so the workflow downloads the\n bytes it selected or fails - it never silently downloads newer ones. An entity\n reaching the input port without an entity tag is fetched unconditionally.\n* **The error handling decides whether one failure ends the run**, but the run\n ends in an error either way. Continuing only means the remaining objects are\n fetched first.\n* **The downloaded files are not cleaned up by the task.** The temporary\n directory it writes into stays on the node until whatever clears its temporary\n storage does, so a workflow fetching more than the node's disk holds will fill\n it, and fetching the same bucket repeatedly accumulates copies.\n* **The preview stops after ten objects**, so it can miss an object further down\n that a real run would fail on.\n ", + "pluginIcon": "data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiIHN0YW5kYWxvbmU9Im5vIj8+CjxzdmcgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiB2aWV3Qm94PSIwIDAgMjQgMjQiIGZpbGw9Im5vbmUiIHN0cm9rZT0iY3VycmVudENvbG9yIiBzdHJva2Utd2lkdGg9IjEuNiIgc3Ryb2tlLWxpbmVjYXA9InJvdW5kIiBzdHJva2UtbGluZWpvaW49InJvdW5kIj4KICA8ZWxsaXBzZSBjeD0iMTIiIGN5PSI1IiByeD0iNyIgcnk9IjIuNiIvPgogIDxwYXRoIGQ9Ik01IDV2Ni40YzAgMS40NCAzLjEzIDIuNiA3IDIuNnM3LTEuMTYgNy0yLjZWNSIvPgogIDxwYXRoIGQ9Ik0xMiAxNS42djUuNiIvPgogIDxwYXRoIGQ9Ik05LjIgMTguNCAxMiAyMS4ybDIuOC0yLjgiLz4KPC9zdmc+Cg==", + "properties": { + "endpoint_url": { + "name": "endpoint_url", + "title": "Endpoint URL", + "description": "The address of the service, e.g. `https://minio.example.org`. Leave empty for Amazon S3, which is then addressed by region.", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "region": { + "name": "region", + "title": "Region", + "description": "The region the bucket lives in. Amazon S3 needs the bucket's own region; most other implementations accept any value and ignore it.", + "type": "string", + "parameterType": "string", + "value": "us-east-1", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "credentials": { + "name": "credentials", + "title": "Credentials", + "description": "Where the credentials come from.", + "type": "string", + "parameterType": "string", + "value": "explicit", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "access_key_id": { + "name": "access_key_id", + "title": "Access key ID", + "description": "The access key to sign requests with.", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "secret_access_key": { + "name": "secret_access_key", + "title": "Secret access key", + "description": "The secret belonging to the access key.", + "type": "string", + "parameterType": "password", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "bucket": { + "name": "bucket", + "title": "Bucket", + "description": "The bucket the objects are downloaded from. Leaving it empty replaces the task's own selection with an input port.", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "prefix": { + "name": "prefix", + "title": "Key prefix", + "description": "Downloads only the objects whose key starts with this, e.g. `raw/2026/`. The service applies it. Empty means the whole bucket.", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "regex": { + "name": "regex", + "title": "Regular expression", + "description": "Downloads only the objects whose key matches this regular expression, e.g. `\\.csv$`. It is matched against the whole key rather than against the name alone, and it has to match somewhere in it rather than all of it. Empty means every key.", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "top_level_only": { + "name": "top_level_only", + "title": "Top level only", + "description": "If enabled, only the keys directly below the prefix are considered and everything nested deeper is folded away. The service does the folding, on a `/` delimiter.", + "type": "string", + "parameterType": "boolean", + "value": "false", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "limit": { + "name": "limit", + "title": "Limit", + "description": "The maximum number of objects to download, or 0 for no limit. A run that stops at the limit says so in its report.", + "type": "string", + "parameterType": "Long", + "value": "0", + "advanced": false, + "visibleInDialog": true, + "properties": {} + } + }, + "properties_advanced": { + "session_token": { + "name": "session_token", + "title": "Session token", + "description": "The session token of a temporary credential. Needed only for keys issued by a token service, which expire.", + "type": "string", + "parameterType": "password", + "value": "", + "advanced": true, + "visibleInDialog": true, + "properties": {} + }, + "addressing_style": { + "name": "addressing_style", + "title": "Bucket addressing style", + "description": "How the bucket is named in a request.", + "type": "string", + "parameterType": "string", + "value": "auto", + "advanced": true, + "visibleInDialog": true, + "properties": {} + }, + "ssl_verify": { + "name": "ssl_verify", + "title": "Verify TLS connection", + "description": "If enabled, the certificate of the service is verified. Disable only when connecting to a service with a self-signed or otherwise untrusted certificate.", + "type": "string", + "parameterType": "boolean", + "value": "true", + "advanced": true, + "visibleInDialog": true, + "properties": {} + }, + "on_error": { + "name": "on_error", + "title": "On error", + "description": "What happens when a single object cannot be transferred. Either way the workflow ends in an error.", + "type": "string", + "parameterType": "string", + "value": "stop", + "advanced": true, + "visibleInDialog": true, + "properties": {} + } + }, + "actions": { + "test_connection": { + "label": "Test connection", + "description": "Reach the service, say which identity it signs as and whether the bucket can be listed.", + "icon": null + }, + "preview": { + "label": "Preview (max. 10)", + "description": "List the first objects the current configuration would download.", + "icon": null + } + }, + "required": [], + "distanceMeasureRange": null, + "backendType": "python", + "is_deprecated": false, + "tags": [ + "WorkflowTask", + "PythonPlugin" + ], + "pluginType": "customtask", + "relatedPlugins": [] + }, "cmem_plugin_ssh-Download": { "pluginId": "cmem_plugin_ssh-Download", "title": "Download SSH files", @@ -1414,14 +1605,14 @@ "Uncategorized" ], "main_category": "Uncategorized", - "description": "Download files from a given SSH instance", - "markdownDocumentation": "\nThis workflow task downloads files from a specified SSH instance.\n\nBy providing the hostname, username, port and authentication method, you can specify the\nfolder from which the data should be extracted.\n\nYou can also define a regular expression to include or exclude specific files.\n\nThere is also an option to prevent files in subfolders from being included.\n\n#### Authentication Methods:\n* **Password:** Only the password will be used for authentication. The private key field is\nignored, even if filled.\n* **Key:** The private key will be used for authentication. If the key is encrypted, the password\nwill be used to decrypt it.\n\n#### Error handling modes:\n* **Ignore:** Ignores the permission rights of files and lists downloads all files it has access to.\nSkips folders and files when there is no correct permission.\n* **Warning:** Warns the user about files that the user has no permission rights to. Downloads\nall other files and skips files folder when there is no correct permission.\n* **Error:** Throws an error when there is a single file or folder with incorrect permission rights.\n\n#### Note:\n* If a connection cannot be established within 20 seconds, a timeout occurs.\n* Currently supported key types are: RSA, ECDSA, Ed25519.\n* Setting the maximum amount of workers to more than 1 may cause a Channel Exception when\nthe amount of files is too large\n ", + "description": "Download files from an SSH server.", + "markdownDocumentation": "\nDownloads files from an SSH server into the workflow.\n\nWhat is downloaded depends on the input port. When nothing is connected, the task lists\nthe configured directory itself and downloads the files matching the regular\nexpression. When a file listing is connected, it downloads exactly the paths carried in\nthe `file_name` value of the incoming entities, and the directory, regular expression\nand subfolder settings are then without effect. Either way the files leave the output\nport as file entities, stored in a temporary directory on the eccenca Corporate Memory\nhost from where the following tasks read them.\n\n**List SSH files** produces the listing this task accepts, so a workflow can list\nfirst, narrow the result down and download only what is left. **Upload SSH files**\ncovers the other direction.\n\nThe **Preview results** action lists the first ten files that a run without input would\ndownload, without running the workflow.\n\n#### Caveats\n\n* Only the plain file name of a remote file is kept. Downloaded files of the same name\ncoming from different folders overwrite each other, and just one of them reaches the\noutput port.\n* A file that cannot be read is dropped from the output silently with both Ignore and\nWarning. The warning names the unreadable files seen while listing, which is not\nnecessarily the same set of files.\n* A folder that cannot be read is skipped silently unless the error handling is set to\nError, and the files below it are then never downloaded.\n* The task logs in with the configured credentials only, and offers no key of the\nmachine it runs on. The host key of the server in turn is accepted as presented and\nnever checked against a known hosts list.\n* Establishing the connection fails after 20 seconds.\n ", "pluginIcon": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTQuNSA5Yy0uNiAwLTEgLjUtMSAxdjEuNzVjMCAuNS40IDEgMSAxSDd2Ljc1SDMuNVYxNWg0Yy42IDAgMS0uNSAxLTF2LTEuNzVjMC0uNS0uNC0xLTEtMUg1di0uNzVoMy41Vjl6bTYgMGMtLjYgMC0xIC41LTEgMXYxLjc1YzAgLjUuNCAxIDEgMUgxM3YuNzVIOS41VjE1aDRjLjYgMCAxLS41IDEtMXYtMS43NWMwLS41LS40LTEtMS0xSDExdi0uNzVoMy41Vjl6bTUgMHY2SDE3di0yLjVoMlYxNWgxLjVWOUgxOXYyaC0yVjl6Ii8+PC9zdmc+", "properties": { "hostname": { "name": "hostname", "title": "Hostname", - "description": "Hostname to connect to. Usually in the form of an IP address", + "description": "Host name or IP address of the SSH server.", "type": "string", "parameterType": "string", "value": null, @@ -1432,7 +1623,7 @@ "port": { "name": "port", "title": "Port", - "description": "The port on which the connection will be tried on. Default is 22.", + "description": "TCP port the SSH server listens on.", "type": "string", "parameterType": "Long", "value": "22", @@ -1443,7 +1634,7 @@ "username": { "name": "username", "title": "Username", - "description": "The username with which a connection will be instantiated.", + "description": "Account to log in as.", "type": "string", "parameterType": "string", "value": null, @@ -1454,7 +1645,7 @@ "authentication_method": { "name": "authentication_method", "title": "Authentication method", - "description": "The method that is used to connect to the SSH server.", + "description": "How the task authenticates against the server.", "type": "string", "parameterType": "string", "value": "password", @@ -1465,7 +1656,7 @@ "private_key": { "name": "private_key", "title": "Private key", - "description": "Your private key to connect via SSH.", + "description": "Private key in PEM format, used when the authentication method is Key. RSA, ECDSA and Ed25519 keys are supported.", "type": "string", "parameterType": "password", "value": "", @@ -1476,7 +1667,7 @@ "password": { "name": "password", "title": "Password", - "description": "Depending on your authentication method this will either be used toconnect via password to SSH, or to decrypt the SSH private key", + "description": "Password of the account, or the passphrase of the private key when the authentication method is Key.", "type": "string", "parameterType": "password", "value": "", @@ -1487,7 +1678,7 @@ "path": { "name": "path", "title": "Path", - "description": "The currently selected path within your SSH instance. Auto-completion starts from user home folder, use '..' for parent directory or '/' for root directory.", + "description": "Remote directory the files are downloaded from, and ignored when a file listing arrives on the input port. Autocompletion starts in the home directory of the account, use '..' for the parent directory or '/' for the root directory.", "type": "string", "parameterType": "string", "value": "", @@ -1497,8 +1688,8 @@ }, "error_handling": { "name": "error_handling", - "title": "Error handling for missing permissions.", - "description": "A choice on how to handle errors concerning the permissions rights.When choosing 'ignore' all files get skipped if the current user has correct permission rights.When choosing 'warning' all files get downloaded however there will be a mention that some of the files are not under the users permissionsif there are any and these get skipped.When choosing 'error' the files will not get downloaded if thereis even a single file the user has no access to.", + "title": "Error handling", + "description": "How the task reacts to files and folders the account cannot read.", "type": "string", "parameterType": "string", "value": "error", @@ -1509,7 +1700,7 @@ "no_subfolder": { "name": "no_subfolder", "title": "No subfolder", - "description": "When this flag is set, only files from the current directory will be downloaded.", + "description": "If enabled, only the given directory is searched and its subfolders are left out.", "type": "string", "parameterType": "boolean", "value": "false", @@ -1520,7 +1711,7 @@ "regex": { "name": "regex", "title": "Regular expression", - "description": "A regular expression used to define which files will get downloaded.", + "description": "Regular expression a file name has to match completely to be downloaded. It is matched against the plain file name, not against the path.", "type": "string", "parameterType": "string", "value": "^.*$", @@ -1532,8 +1723,8 @@ "properties_advanced": { "max_workers": { "name": "max_workers", - "title": "Maximum amount of workers.", - "description": "Determines the amount of workers used for concurrent thread execution of the task. Default is 1, maximum is 32. Note that too many workers can cause a ChannelException.", + "title": "Maximum number of workers", + "description": "Number of threads the Preview results action uses to list subfolders in parallel, from 1 to 32. A workflow run always lists with a single thread. Too many parallel channels make some servers refuse them with a ChannelException.", "type": "string", "parameterType": "Long", "value": "1", @@ -1545,7 +1736,7 @@ "actions": { "preview_results": { "label": "Preview results (max. 10)", - "description": "Lists 10 files as a preview.", + "description": "Connect to the server and list the first ten matching files.", "icon": null } }, @@ -1570,8 +1761,8 @@ "Uncategorized" ], "main_category": "Uncategorized", - "description": "Evaluates a template on a sequence of entities. Can be used after a transformation or directly after datasets that output a single table, such as CSV or Excel.", - "markdownDocumentation": "\nThe template operator supports the Jinja templating language. Documentation about Jinja can be found in the official [Template Designer Documentation](https://jinja.palletsprojects.com/en/2.11.x/templates/).\n\nNote that support for RDF properties is limited, because Jinja does not support some special characters (in particula colons) in variable names. This makes it impractical to access RDF properties. For this reason, the transformation that precedes the template operator needs to make sure that it generates attributes that are valid Jinja variable names.\n\n## Default evaluation\n\nBy default, the template is evaluated separately for each entity.\nFor each input entity, a output entity is generated that provides a single output attribute, which contains the evaluated template.\n\n*Limitation*: For the default evaluation, accessing nested paths is not supported. If the preceding transformation contains hierarchical mappings, only the attributes from the root mapping can be accessed.\n\n## Full evaluation\n\nIf 'full evaluation' is enabled, the entire input set will be evaluated at once.\n\nThe entities variable will contain all input entities and can be iterated over: \n\n {% for entity in entities %}\n {{entity.property}}\n {% endfor %}\n\nA single output entity will be generated that contains the evaluated template.\n\nIf the input entities are hierarchical (typically the case if the input transformation is hierarchical), each entity will be hierarchical as well.\n\nExample iterating over an sequence of books that each contains a list of chapters:\n\n {% for book in entities %}\n Book {{book.title}}\n {% for chapter in book.chapter %}\n Chapter {{chapter.chapterNumber}}\n {% endfor %}\n {% endfor %}\n\nIn this example, the child mapping defines a `chapter` target property from which it is accessible from the root entities. If the child mapping allows multiple entities, the value of the property will be a list of entities.\n", + "description": "Evaluates a template on a sequence of entities. Can be used after a transformation or directly after datasets that output a single table, such as CSV or Excel. A template that uses no input variables can be evaluated without an input.", + "markdownDocumentation": "\nThe template operator supports the Jinja templating language. Documentation about Jinja can be found in the official [Template Designer Documentation](https://jinja.palletsprojects.com/en/2.11.x/templates/).\n\nNote that support for RDF properties is limited, because Jinja does not support some special characters (in particula colons) in variable names. This makes it impractical to access RDF properties. For this reason, the transformation that precedes the template operator needs to make sure that it generates attributes that are valid Jinja variable names.\n\n\n## Default evaluation\n\nBy default, the template is evaluated separately for each entity.\nFor each input entity, a output entity is generated that provides a single output attribute, which contains the evaluated template.\n\n*Limitation*: For the default evaluation, accessing nested paths is not supported. If the preceding transformation contains hierarchical mappings, only the attributes from the root mapping can be accessed.\n\n## Full evaluation\n\nIf 'full evaluation' is enabled, the entire input set will be evaluated at once.\n\nThe entities variable will contain all input entities and can be iterated over: \n\n {% for entity in entities %}\n {{entity.property}}\n {% endfor %}\n\nA single output entity will be generated that contains the evaluated template.\n\nIf the input entities are hierarchical (typically the case if the input transformation is hierarchical), each entity will be hierarchical as well.\n\nExample iterating over a sequence of books that each contains a list of chapters:\n\n {% for book in entities %}\n Book {{book.title}}\n {% for chapter in book.chapter %}\n Chapter {{chapter.chapterNumber}}\n {% endfor %}\n {% endfor %}\n\nIn this example, the child mapping defines a `chapter` target property from which it is accessible from the root entities. If the child mapping allows multiple entities, the value of the property will be a list of entities.\n\n## Static evaluation\n\nThe template operator can be used without any inputs. The input port will only appear if a variable referencing an input is used or if the 'Forward input attributes' option is enabled.\nE.g. the use of the 'entities' variable in case of the full evaluation would require the connection of an input. \n\nVariables with the scopes 'execution', 'project' and 'global' can be used without the need for an input task/dataset.\n\nWithout an input port, the template is evaluated once and a single output entity is generated. An input task/dataset that is still connected to the operator is not read.\n", "pluginIcon": null, "properties": { "template": { @@ -1762,6 +1953,109 @@ "pluginType": "customtask", "relatedPlugins": [] }, + "cmem_plugin_querycatalog-ExecuteCatalogQuery": { + "pluginId": "cmem_plugin_querycatalog-ExecuteCatalogQuery", + "title": "Execute Catalog Query", + "categories": [ + "Uncategorized" + ], + "main_category": "Uncategorized", + "description": "Run a query stored in a query catalog and hand on its result.", + "markdownDocumentation": "\nThis task runs a query that is stored in a query catalog, so a workflow references the\nstored definition instead of carrying a second copy of it.\n\n## What flows through the ports\n\nWhat leaves the task follows the type of the query. A SELECT query produces one entity\nper result row with one value per projected variable, and a variable the query left\nunbound produces no value rather than an empty string. An ASK query produces a single\nentity carrying `true` or `false`. A CONSTRUCT or DESCRIBE query produces its\nstatements as quad entities with no graph set, ready for any task that consumes quads.\nAn update query produces nothing at all: the task is a terminal step and hands nothing\non.\n\nA query that contains placeholders turns their names into input fields, so the\npreceding task can be mapped onto them, and the query is then run once per incoming\nentity with the results of all of them arriving together. A query without placeholders\nhas no input and runs on its own.\n\n## Where the task sits\n\n**Get Catalog Query** and **List Catalog Queries** describe queries without running\nthem, and are the way to find the query this task should be pointed at.\n\n## Caveats\n\n- **The stored query is read again on every execution.** Editing it in the query\n catalog changes what runs next time, which is the point of the task - but it also\n means a query that gained a placeholder since the workflow was wired fails, naming\n what changed, rather than running with a hole in it.\n- **The query kind is a safety gate, not only a filter.** A query that has become an\n update is refused while the task is set to read queries, so editing the catalog\n cannot turn a reading step into a writing one.\n- **The whole result is held in memory.** The store sends it in one response, so a\n query returning very many rows is limited by what the DataIntegration process can\n hold.\n- **Continuing past a failed input entity can leave a write half applied.** Entities\n are run one after another against the store and nothing rolls an earlier one back.\n", + "pluginIcon": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAzMiAzMiIgZmlsbD0iY3VycmVudENvbG9yIj4KICA8cGF0aCBkPSJNMTkuMTYsMTkuOTVsMy45NSwxLjk3NWwtMy45NSwxLjk3NVoiLz4KICA8cGF0aCBkPSJNMjYuNDg4OCwyMS41MTNBNi44MjU2LDYuODI1NiwwLDAsMCwyMC4zNDUsMTcuMTg1YTYuODI1Niw2LjgyNTYsMCwwLDAsLTYuMTQzOCw0LjMyOEwxNC4wMjUsMjEuOTI1bDAuMTc2MiwwLjQxMkE2LjgyNTYsNi44MjU2LDAsMCwwLDIwLjM0NSwyNi42NjVhNi44MjU2LDYuODI1NiwwLDAsMCw2LjE0MzgsLTQuMzI4TDI2LjY2NSwyMS45MjVaTTIwLjM0NSwyNS4wODVhMy4xNiwzLjE2LDAsMSwxLDMuMTYsLTMuMTZBMy4xNjM2LDMuMTYzNiwwLDAsMSwyMC4zNDUsMjUuMDg1WiIvPgogIDxjaXJjbGUgY3g9IjkuMjg1IiBjeT0iOS4yODUiIHI9IjAuNzkiLz4KICA8Y2lyY2xlIGN4PSI5LjI4NSIgY3k9IjE1LjYwNSIgcj0iMC43OSIvPgogIDxjaXJjbGUgY3g9IjkuMjg1IiBjeT0iMjEuOTI1IiByPSIwLjc5Ii8+CiAgPHBhdGggZD0iTTYuOTE1LDE5LjU1NWg1LjUzVjE3Ljk3NUg2LjkxNVYxMy4yMzVIMTkuNTU1djIuMzdoMS41OFY2LjkxNWExLjU4LDEuNTgsMCwwLDAsLTEuNTgsLTEuNThINi45MTVBMS41OCwxLjU4LDAsMCwwLDUuMzM1LDYuOTE1VjI0LjI5NWExLjU4LDEuNTgsMCwwLDAsMS41OCwxLjU4aDUuNTNWMjQuMjk1SDYuOTE1Wk02LjkxNSw2LjkxNUgxOS41NTV2NC43NEg2LjkxNVoiLz4KPC9zdmc+Cg==", + "properties": { + "catalog_graph": { + "name": "catalog_graph", + "title": "Catalog Graph", + "description": "The graph the query is read from. The list offers the query catalogs first, then the shape catalogs, then any other graph that holds queries, with the number of read and write queries each one contains.", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "query_kind": { + "name": "query_kind", + "title": "Query Kind", + "description": "Which half of the catalog the query list offers. It also decides what this task is allowed to run: a query that has become an update since the task was configured is refused while this is set to read queries.", + "type": "string", + "parameterType": "string", + "value": "read", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "query": { + "name": "query", + "title": "Query", + "description": "The query to use, picked by its label. Choose the catalog graph and the query kind first; changing either of them clears this field.", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "context": { + "name": "context", + "title": "Context", + "description": "", + "type": "string", + "parameterType": "PluginContext", + "value": "", + "advanced": false, + "visibleInDialog": false, + "properties": {} + } + }, + "properties_advanced": { + "timeout": { + "name": "timeout", + "title": "Timeout", + "description": "How many milliseconds the store may spend on one execution before it is aborted. Zero leaves the decision to the store, which is the right choice unless a query has been seen to run away.", + "type": "string", + "parameterType": "Long", + "value": "0", + "advanced": true, + "visibleInDialog": true, + "properties": {} + }, + "on_error": { + "name": "on_error", + "title": "On error", + "description": "What happens when the query fails for a single input entity. Either way the workflow ends in an error. Note that continuing through a write query can leave part of its changes applied: the entities are run one after another and nothing rolls them back.", + "type": "string", + "parameterType": "string", + "value": "stop", + "advanced": true, + "visibleInDialog": true, + "properties": {} + } + }, + "actions": { + "preview_query": { + "label": "Preview query", + "description": "Show the query that this task would run.", + "icon": null + }, + "open_in_editor": { + "label": "Open in query editor", + "description": "Link the chosen query in the query editor.", + "icon": null + } + }, + "required": [], + "distanceMeasureRange": null, + "backendType": "python", + "is_deprecated": false, + "tags": [ + "WorkflowTask", + "PythonPlugin" + ], + "pluginType": "customtask", + "relatedPlugins": [] + }, "cmem_plugin_ssh-Execute": { "pluginId": "cmem_plugin_ssh-Execute", "title": "Execute commands via SSH", @@ -1769,14 +2063,14 @@ "Uncategorized" ], "main_category": "Uncategorized", - "description": "Execute commands on a given SSH instance.", - "markdownDocumentation": "\nThis workflow task executes commands on a given SSH instance.\n\nBy providing the hostname, username, port and authentication method, you can specify the\nfolder in which the command should be executed in.\n\n#### Input Methods:\n* **No input:** The command will be executed with no input attached to the plugin. Stdin\nis non-existent in this case.\n* **File input:** The command will be executed with the stdin being represented by the\nfiles that are connected via the input port of the plugin. This also allows for looping\nover multiple files executing the same command over them.\n\n\n#### Output Methods:\n* **Structured process output:** The output will produce entities with its own schema including\nthe stdout and stderr as well as the exit code to confirm the execution of the command.\n* **File output:** The stdout will be converted into a file a be provided for further use.\n* **No output:** The output port will be closed.\n\n#### Authentication Methods:\n* **Password:** Only the password will be used for authentication. The private key field is\nignored, even if filled.\n* **Key:** The private key will be used for authentication. If the key is encrypted, the password\nwill be used to decrypt it.\n\n#### Note:\n* If a connection cannot be established within 20 seconds, a timeout occurs.\n* Currently supported key types are: RSA, ECDSA, Ed25519.\n ", + "description": "Execute a command on an SSH server.", + "markdownDocumentation": "\nRuns a command on an SSH server and collects what it produces.\n\nThe input and output methods decide the shape of the task. With file input, an input\nport accepts file entities and the command runs once per incoming file, with the\ncontent of that file on its standard input; without it, the command runs exactly once.\nWhat leaves the task is either one entity per run, carrying the exit code, the standard\noutput and the standard error, or one file per run holding the raw standard output, or\nnothing at all, in which case the task ends the branch of the workflow it sits in.\n\n**Upload SSH files** and **Download SSH files** move the files a command works on, so a\ncommon chain is to upload files, run a command over them and download the result.\n\n#### Caveats\n\n* The exit code is never inspected. A failing command leaves the task successful and\nthe workflow running, and the failure shows up only in the emitted exit code and\nstandard error. With file output, both are dropped and the standard output is kept\nalone.\n* The command is sent as it is written and interpreted by the login shell of the\naccount, without any escaping or checking, and it runs with every right that account\nhas.\n* The command runs in whatever directory the account lands in on login. The configured\ndirectory is not entered.\n* With file input, each incoming file is read into memory as a whole before it is sent.\n* The task logs in with the configured credentials only, and offers no key of the\nmachine it runs on. The host key of the server in turn is accepted as presented and\nnever checked against a known hosts list.\n* Establishing the connection fails after 20 seconds.\n ", "pluginIcon": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTQuNSA5Yy0uNiAwLTEgLjUtMSAxdjEuNzVjMCAuNS40IDEgMSAxSDd2Ljc1SDMuNVYxNWg0Yy42IDAgMS0uNSAxLTF2LTEuNzVjMC0uNS0uNC0xLTEtMUg1di0uNzVoMy41Vjl6bTYgMGMtLjYgMC0xIC41LTEgMXYxLjc1YzAgLjUuNCAxIDEgMUgxM3YuNzVIOS41VjE1aDRjLjYgMCAxLS41IDEtMXYtMS43NWMwLS41LS40LTEtMS0xSDExdi0uNzVoMy41Vjl6bTUgMHY2SDE3di0yLjVoMlYxNWgxLjVWOUgxOXYyaC0yVjl6Ii8+PC9zdmc+", "properties": { "hostname": { "name": "hostname", "title": "Hostname", - "description": "Hostname to connect to. Usually in the form of an IP address", + "description": "Host name or IP address of the SSH server.", "type": "string", "parameterType": "string", "value": null, @@ -1787,7 +2081,7 @@ "port": { "name": "port", "title": "Port", - "description": "The port on which the connection will be tried on. Default is 22.", + "description": "TCP port the SSH server listens on.", "type": "string", "parameterType": "Long", "value": "22", @@ -1798,7 +2092,7 @@ "username": { "name": "username", "title": "Username", - "description": "The username with which a connection will be instantiated.", + "description": "Account to log in as.", "type": "string", "parameterType": "string", "value": null, @@ -1809,7 +2103,7 @@ "authentication_method": { "name": "authentication_method", "title": "Authentication method", - "description": "The method that is used to connect to the SSH server.", + "description": "How the task authenticates against the server.", "type": "string", "parameterType": "string", "value": "password", @@ -1820,7 +2114,7 @@ "private_key": { "name": "private_key", "title": "Private key", - "description": "Your private key to connect via SSH.", + "description": "Private key in PEM format, used when the authentication method is Key. RSA, ECDSA and Ed25519 keys are supported.", "type": "string", "parameterType": "password", "value": "", @@ -1831,7 +2125,7 @@ "password": { "name": "password", "title": "Password", - "description": "Depending on your authentication method this will either be used toconnect via password to SSH, or to decrypt the SSH private key", + "description": "Password of the account, or the passphrase of the private key when the authentication method is Key.", "type": "string", "parameterType": "password", "value": "", @@ -1842,7 +2136,7 @@ "path": { "name": "path", "title": "Path", - "description": "The currently selected path within your SSH instance. Auto-completion starts from user home folder, use '..' for parent directory or '/' for root directory.", + "description": "Remote directory, kept with the task but not entered before the command runs. Autocompletion starts in the home directory of the account, use '..' for the parent directory or '/' for the root directory.", "type": "string", "parameterType": "string", "value": "", @@ -1853,7 +2147,7 @@ "input_method": { "name": "input_method", "title": "Input method", - "description": "Parameter to decide whether files will be used as stdin or no input is needed. If 'File input' is chosen, the input port will open for all entities withthe FileEntitySchema.", + "description": "Whether incoming files are fed to the command, which also decides whether the task has an input port.", "type": "string", "parameterType": "string", "value": null, @@ -1864,7 +2158,7 @@ "output_method": { "name": "output_method", "title": "Output method", - "description": "Parameter to decide which type of output the user wants. This can be either no output, a structured process output with its own schema, or a file based output", + "description": "What the task emits for each run of the command, which also decides whether the task has an output port.", "type": "string", "parameterType": "string", "value": null, @@ -1875,7 +2169,7 @@ "command": { "name": "command", "title": "Command", - "description": "The command that will be executed on the SSH instance. When the inputmethod is set to 'File input', the command will be executed over these files.", + "description": "Command line executed on the server.", "type": "string", "parameterType": "string", "value": "ls", @@ -1886,7 +2180,7 @@ "timeout": { "name": "timeout", "title": "Timeout", - "description": "A timeout for the executed command.", + "description": "Seconds the task waits for the command to send data before it gives up and fails. Zero waits for as long as the command takes.", "type": "string", "parameterType": "Long", "value": "0", @@ -1920,14 +2214,14 @@ "Uncategorized" ], "main_category": "Uncategorized", - "description": "Send instructions (prompt) to an LLM and process the result.", - "markdownDocumentation": "\n## Overview\n\nThis plugin executes Large Language Model (LLM) instructions over entity collections, enabling\nAI-powered text generation, analysis, and transformation tasks within Corporate Memory workflows.\n\n## Core Functionality\n\n- **LLM Integration**: Supports OpenAI API, Azure OpenAI, and OpenAI-compatible endpoints\n (Anthropic Claude, OpenRouter, etc.)\n- **Entity Processing**: Processes entities individually or in batches with configurable\n concurrency\n- **Template System**: Uses Jinja2 templates for dynamic prompt generation from entity data\n- **File Inputs**: Optionally accept files (images and documents) from CMEM as multimodal\n LLM inputs\n- **Output Formats**: Supports text, JSON mode, and structured outputs with Pydantic schemas\n- **Performance Optimization**: Includes batching, rate limiting, and async processing for\n high-throughput scenarios\n\n## Input/Output Behavior\n\nAfter processing, each entity receives an additional path (default: `_instruction_output`)\ncontaining the LLM response. For TEXT and JSON_MODE output formats, the response is stored\nin this path. For STRUCTURED_OUTPUT, the Pydantic model fields are directly added to the\nentity (the `_instruction_output` path is not used).\n\nInput/output ports are automatically configured based on template variables and the\n**Enable File Inputs** option:\n\n- **No placeholders**: No input ports required\n- **With placeholders**: Single input port created for entity data\n- **Schema handling**: Fixed schemas when using specific entity paths, flexible schemas otherwise\n- **File Inputs enabled**: Single input port accepting `FileEntitySchema` entities (template\n variables on the entity are ignored)\n\n## Template System\n\nUses Jinja2 templating for dynamic prompts:\n\n```jinja2\n{{ entity }} # Entire entity as JSON\n{{ entity.name }} # Specific entity property\n```\n\nThe following template processing rules are implemented:\n\n1. **Variable Extraction**: Automatically detects template variables to configure input ports\n2. **Entity Iteration**: Processes entities from the single input port individually\n3. **Single Entity Context**: Each entity is processed independently with its own template context\n\n## File Inputs\n\nEnable **Enable File Inputs** to send files (images or documents) from CMEM to the LLM as\nmultimodal input. When activated:\n\n- The input port switches to `FileEntitySchema`, accepting file entities.\n- Each file is read from the project, base64-encoded, and embedded into the chat message\n as a `data:` URL.\n- The `message_template` parameter is ignored and the message structure is selected automatically\n by MIME type:\n - **Images** (e.g. PNG, JPEG) are sent as `image_url` content parts.\n - **Documents** (e.g. PDF, spreadsheets, rich documents, presentations, text and code) are\n sent as `file` content parts with the original filename preserved.\n- If no MIME type is given, it is guessed by filename extension.\n- The `developer_prompt_template` and `instruct_prompt_template` is sent alongside the file.\n Entity-derived template variables (e.g. `{{ entity.name }}`) are not available in this mode.\n- For **TEXT** and **JSON_MODE** outputs, the output entity additionally carries\n `_filename` and `_mimetype` paths so downstream tasks can see what the model saw.\n **STRUCTURED_OUTPUT** keeps the schema pure and does not add these paths.\n\nNote: Multimodal capability depends on the chosen model. Use a vision-capable model for\nimages and a model that accepts file/document inputs for documents.\n\nFor more details and all available MIME types see the official documentation:\n- [Images and vision](https://developers.openai.com/api/docs/guides/images-vision?api-mode=chat)\n- [File inputs](https://developers.openai.com/api/docs/guides/file-inputs?api-mode=chat)\n\n## Output Formats\n\n1. **Text Output (Default)** - Standard LLM text responses for general-purpose tasks.\n2. **JSON Mode** - Ensures valid JSON output format. Add JSON structure requirements\n to your prompt template.\n3. **Structured Output** - Uses Pydantic schemas for type-safe, validated responses:\n\n```python\nfrom pydantic import BaseModel\n\nclass StructuredOutput(BaseModel):\n title: str\n summary: str\n keywords: list[str]\n confidence_score: float\n```\n\n## Performance Features\n\nParallel Processing:\n- **Concurrent Requests**: Configurable semaphore-controlled API calls\n- **Batch Processing**: Entities processed in configurable batch sizes\n- **Rate Limiting**: Optional delays between requests\n- **Memory Optimization**: Streaming processing with generator patterns\n\nError Handling:\n- **Graceful Degradation**: Continue processing on API errors (configurable)\n- **Detailed Logging**: Comprehensive error reporting and debugging information\n- **Workflow Integration**: Proper cancellation support and progress reporting\n\n## API Compatibility\n\nSupported Providers:\n- **OpenAI**: Direct API access with full feature support\n- **Azure OpenAI**: Enterprise Azure-hosted services with API versioning\n- **OpenAI-Compatible**: Anthropic Claude, OpenRouter, local models, and other compatible endpoints\n\nAuthentication:\n- **API Keys**: Secure password-type parameters for API authentication\n- **Azure Integration**: Supports Azure OpenAI API versioning and endpoint configuration\n- **Flexible Endpoints**: Custom base URLs for various providers\n\n## Advanced Configuration\n\n### Message Templates\nCustomize the conversation structure beyond simple prompts:\n\n```json\n[\n {\"role\": \"system\", \"content\": \"You are a data analyst.\"},\n {\"role\": \"user\", \"content\": \"{{ instruction_prompt }}\"}\n]\n```\n\n### Performance Tuning\n- **Temperature Control**: Adjust creativity vs. determinism (0.0-2.0)\n- **Timeout Management**: Request-level timeout configuration\n- **Concurrency Limits**: Prevent rate limiting with request throttling\n- **Batch Optimization**: Balance memory usage vs. throughput\n\n## Best Practices\n\n1. **Schema Design**: Use specific entity paths in templates for fixed schemas\n2. **Error Strategy**: Enable error continuation for large datasets\n3. **Performance**: Adjust concurrency and batch size based on API limits\n4. **Templates**: Design prompts with clear instructions and expected outputs\n5. **Testing**: Start with small entity sets to validate templates and outputs\n\nFor detailed prompting guidance, see [OpenAI's Text Generation Guide](https://platform.openai.com/docs/guides/text?api-mode=chat).\n", + "description": "Prompt a large language model once per entity and add the answer to the entity.", + "markdownDocumentation": "\nSends a prompt to a large language model once for every entity it receives, and adds the\nmodel's answer to that entity. The prompt is a Jinja template rendered against the entity, so\nevery entity gets a prompt of its own. Requests go over the OpenAI chat completions protocol,\nwhich covers OpenAI itself, Azure OpenAI and any service speaking the same protocol -\nAnthropic, OpenRouter, or a model hosted locally. With no endpoint configured, the task uses\nthe built-in model proxy of the eccenca Corporate Memory deployment it runs in.\n\n## Ports\n\nEntities arrive on one input port and leave, enriched, on one output port. The shape of both\nfollows from the instruction prompt alone; variables written into the developer prompt or into\nthe messages template are not looked at.\n\n- **A prompt without variables** is the same for every entity, so there is nothing to read.\n The task then has no input port at all and emits a single entity.\n- **`{{ entity }}`** renders the whole entity, whose paths cannot be known beforehand. Both\n ports are flexible, and everything that arrives is passed through to the output.\n- **`{{ entity.title }}`** names one path. The input port then asks for exactly the paths the\n prompt names, and the output port carries them through beside the answer.\n\n`entity` is the only variable name a prompt may introduce. Any other one is rejected while the\ntask is being configured rather than at run time.\n\nThe answer is added on a path of its own. Under structured output it is the fields of the\nschema you supply that become paths instead, and a model nested inside that schema becomes a\nsub-entity rather than a path. Switching the task to file input replaces the input side\nentirely: the port then takes file entities, one file is sent per request, and there are no\nentity paths left for a prompt to read.\n\n## Where it fits\n\nThe task sits between whatever produces entities - a dataset, a knowledge graph, an earlier\ntransformation - and whatever consumes them, and leaves what it passes through untouched.\n**Create Embeddings** is its counterpart: that task turns text into vectors for retrieval\nrather than generating new text, and the two often appear in one workflow, one finding the\nentities worth looking at and the other writing about them.\n\n## Caveats\n\nThe model is asked once per entity, so a run costs one request per entity, and how long it\ntakes is governed by how many of those may be in flight at a time.\n\nCancelling a workflow takes effect between batches. Requests already sent are finished first.\n\nOnly failures reported by the API count as API errors. A prompt that cannot be rendered, a\nmessages template that is not valid JSON and a file that cannot be read stop the workflow\nwhatever the error setting says.\n\nWhere API errors are not raised, the error text is written to the answer path and the workflow\ncarries on, so a later task can consume a failure message as though it were an answer. Under\nstructured output that is worse than it sounds: the declared output schema has no answer path,\nthe error text is written to one regardless, and the values then no longer line up with the\nfields the schema promises.\n\nJSON mode guarantees only that the answer parses as JSON, never that it has the structure you\nhad in mind - that still has to be described in the prompt. Structured output does constrain\nthe shape, but only against providers and models that implement it.\n", "pluginIcon": "data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiIHN0YW5kYWxvbmU9Im5vIj8+CjwhLS0gVXBsb2FkZWQgdG86IFNWRyBSZXBvLCB3d3cuc3ZncmVwby5jb20sIEdlbmVyYXRvcjogU1ZHIFJlcG8gTWl4ZXIgVG9vbHMgLS0+Cgo8c3ZnCiAgIGZpbGw9IiMwMDAwMDAiCiAgIGhlaWdodD0iODAwcHgiCiAgIHdpZHRoPSI4MDBweCIKICAgdmVyc2lvbj0iMS4xIgogICBpZD0iTGF5ZXJfMSIKICAgdmlld0JveD0iMCAwIDUxMi4yNTYgNTEyLjI1NiIKICAgeG1sOnNwYWNlPSJwcmVzZXJ2ZSIKICAgc29kaXBvZGk6ZG9jbmFtZT0iZXhlY3V0ZV9pbnN0cnVjdGlvbi5zdmciCiAgIGlua3NjYXBlOnZlcnNpb249IjEuMS4yIChiOGUyNWJlOCwgMjAyMi0wMi0wNSkiCiAgIHhtbG5zOmlua3NjYXBlPSJodHRwOi8vd3d3Lmlua3NjYXBlLm9yZy9uYW1lc3BhY2VzL2lua3NjYXBlIgogICB4bWxuczpzb2RpcG9kaT0iaHR0cDovL3NvZGlwb2RpLnNvdXJjZWZvcmdlLm5ldC9EVEQvc29kaXBvZGktMC5kdGQiCiAgIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIKICAgeG1sbnM6c3ZnPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+PGRlZnMKICAgaWQ9ImRlZnM4MzMiIC8+PHNvZGlwb2RpOm5hbWVkdmlldwogICBpZD0ibmFtZWR2aWV3ODMxIgogICBwYWdlY29sb3I9IiNmZmZmZmYiCiAgIGJvcmRlcmNvbG9yPSIjNjY2NjY2IgogICBib3JkZXJvcGFjaXR5PSIxLjAiCiAgIGlua3NjYXBlOnBhZ2VzaGFkb3c9IjIiCiAgIGlua3NjYXBlOnBhZ2VvcGFjaXR5PSIwLjAiCiAgIGlua3NjYXBlOnBhZ2VjaGVja2VyYm9hcmQ9IjAiCiAgIHNob3dncmlkPSJmYWxzZSIKICAgaW5rc2NhcGU6em9vbT0iMS4wNTg3NSIKICAgaW5rc2NhcGU6Y3g9IjQwMCIKICAgaW5rc2NhcGU6Y3k9IjQwMCIKICAgaW5rc2NhcGU6d2luZG93LXdpZHRoPSIxOTIwIgogICBpbmtzY2FwZTp3aW5kb3ctaGVpZ2h0PSIxMDI3IgogICBpbmtzY2FwZTp3aW5kb3cteD0iMTcyOCIKICAgaW5rc2NhcGU6d2luZG93LXk9IjI1IgogICBpbmtzY2FwZTp3aW5kb3ctbWF4aW1pemVkPSIxIgogICBpbmtzY2FwZTpjdXJyZW50LWxheWVyPSJMYXllcl8xIiAvPgo8ZwogICB0cmFuc2Zvcm09Im1hdHJpeCgwLjc1OTc0OTc5LDAsMCwwLjc1OTc0OTc5LDYwLjc3NTE1OCw2MS41MzQ2MTkpIgogICBpZD0iZzgyOCI+Cgk8ZwogICBpZD0iZzgyNiI+CgkJPHBhdGgKICAgZD0ibSA1MDYuODg1LDMzNy4wMDEgYyAtNi41MTMsLTYuNTEzIC0xMS42NzIsLTEzLjkwOCAtMTUuNjI4LC0yMi4wODMgLTcuNzE1LC0xNS45NDIgLTEwLjI3MywtMzMuMDA3IC05LjczLC00Ny41NjEgMjAuODExLC0yNi4xMDUgMzEuNjA3LC01NC4yNDYgMzEuNjA3LC04NS45MzkgMCwtODguNDQxIC04NC4xMSwtMTQ5LjMzMyAtMTkyLC0xNDkuMzMzIC05MS42MzEsMCAtMTY3LjAyLDQ0LjE5NCAtMTg2Ljc5LDExMy4wOTQgQyA1Ni4xODIsMTYzLjAxMSAxLjEyMywyMTYuNTI1IDEuMTIzLDI4OC4wNzcgYyAwLDMxLjY5NyAxMC44MDIsNTkuODYxIDMxLjYwNyw4NS45NDMgMC41NDIsMTQuNTUzIC0yLjAxNiwzMS42MTYgLTkuNzMsNDcuNTU2IC0zLjk1Niw4LjE3NSAtOS4xMTYsMTUuNTcgLTE1LjYyOCwyMi4wODMgLTE1Ljg5NCwxNS44OTQgMC40MjcsNDIuMzg5IDIxLjc3MiwzNS4zNDMgMTAuMTU2LC0zLjM1MyAyNy41MzYsLTEwLjA2NCA1MC40NTUsLTE5LjMzOSAxLjgxMiwtMC43MzQgMS44MTIsLTAuNzM0IDMuNjIxLC0xLjQ2OCAxMi4xMDgsLTQuOTE4IDI0Ljg2LC0xMC4xODEgMzcuNDcyLC0xNS40NDQgNS4xMTksLTIuMTM2IDkuNDQ5LC0zLjk1MiAxMi43NDcsLTUuMzQgaCA1OS42ODUgYyA4OS40OTIsMCAxNjIuNjE3LC00MS44OTcgMTg0Ljk1NywtMTA2LjY1OCBoIDIuNzM4IGMgMy4yOTgsMS4zODggNy42MjgsMy4yMDQgMTIuNzQ3LDUuMzQgMTIuNjEyLDUuMjYzIDI1LjM2NCwxMC41MjcgMzcuNDcyLDE1LjQ0NCAxLjgwOSwwLjczNCAxLjgwOSwwLjczNCAzLjYyMSwxLjQ2OCAyMi45MTksOS4yNzUgNDAuMjk5LDE1Ljk4NyA1MC40NTUsMTkuMzM5IDIxLjM0Myw3LjA0NiAzNy42NjUsLTE5LjQ0OSAyMS43NzEsLTM1LjM0MyB6IG0gLTMxMy43NjIsNTcuNzQzIGggLTY0IGMgLTIuODU2LDAgLTUuNjgzLDAuNTczIC04LjMxMywxLjY4NiAtMy4yNzEsMS4zODQgLTkuMDUyLDMuODE2IC0xNi41NSw2Ljk0NCAtMTEuNTAyLDQuOCAtMjMuMTE5LDkuNTk5IC0zNC4yMjgsMTQuMTIzIDUuMDA1LC0xNy42MTIgNi4yMDcsLTM0LjkxNiA1LjAzMiwtNTAuNDQxIC0wLjE0NCwtMS45MDggLTAuMzAyLC0zLjM2NCAtMC40MzQsLTQuMzIxIC0wLjU2MSwtNC4wNzUgLTIuMjg4LC03LjkwMiAtNC45NzQsLTExLjAxOCAtMTcuNDY4LC0yMC4yNjkgLTI1Ljg2NywtNDAuNTI1IC0yNS44NjcsLTYzLjY0MSAwLC01MS4zNDkgNDMuODQsLTkxLjQ1MiAxMDguOTg3LC0xMDMuMTcgMC45MjYsLTAuMDM1IDEuODY0LC0wLjEyNiAyLjgxLC0wLjI4NSAxMi41NjUsLTIuMTE1IDI1LjA0MiwtMy4yMDMgMzcuNTQ4LC0zLjIwMyA0Ni4wODMsMCA4NS4zNDgsMTIuNzg3IDExMi4wMzMsMzMuOTc0IDIzLjU0MSwxOC42OTUgMzcuMjksNDMuOTI3IDM3LjI5LDcyLjY4NSAwLDAuODQ1IC0wLjAxNywxLjY4NiAtMC4wNCwyLjUyNSAtMC4xNjQsNC41NTggLTAuNzY0LDkuMzM4IC0xLjgxNiwxNC43MzMgLTAuMTMyLDAuNjc4IC0wLjIyMywxLjM1MiAtMC4yODksMi4wMjQgLTExLjY2MSw1MS4zMzMgLTY5LjgxMSw4Ny4zODUgLTE0Ny4xODksODcuMzg1IHogTSA0NDQuNTk0LDI0NS4wNjggYyAtMi42ODIsMy4xMTUgLTQuNDA3LDYuOTM4IC00Ljk2OCwxMS4wMSAtMC4xMzIsMC45NTcgLTAuMjg5LDIuNDEzIC0wLjQzNCw0LjMyMSAtMS4xNzUsMTUuNTI1IDAuMDI3LDMyLjgyOSA1LjAzMiw1MC40NDEgLTExLjEwOSwtNC41MjQgLTIyLjcyNiwtOS4zMjMgLTM0LjIyOCwtMTQuMTIzIC03LjQ5OCwtMy4xMjkgLTEzLjI3OSwtNS41NiAtMTYuNTUsLTYuOTQ0IC0yLjYzLC0xLjExMyAtNS40NTcsLTEuNjg2IC04LjMxMywtMS42ODYgMCwtNDIuODQyIC0xOS43MzcsLTc5LjIxOSAtNTIuMzUyLC0xMDUuMzQyIC0zNC43MTYsLTI3LjgxIC04NC4wMjUsLTQ0IC0xMzkuNjU4LC00NCAtMy40MTQsMCAtNi44MDMsMC4wNjIgLTEwLjE2OCwwLjE4MyAyMS42NjksLTM4LjY2OSA3My41ODYsLTY0LjE3NCAxMzguMTc4LC02NC4xNzQgODYuNzQ4LDAgMTQ5LjMzMyw0NS4zMSAxNDkuMzMzLDEwNi42NjcgMTBlLTQsMjMuMTA3IC04LjM5Miw0My4zNDQgLTI1Ljg3Miw2My42NDcgeiIKICAgaWQ9InBhdGg4MjQiIC8+Cgk8L2c+CjwvZz4KPC9zdmc+Cg==", "properties": { "model": { "name": "model", - "title": "Instruct Model", - "description": "The identifier of the instruct model to use. Note that some provider do not support a model list endpoint. Just create a custom entry then. Available model IDs for some public providers can be found here: [OpenAI](https://platform.openai.com/docs/models), [Claude](https://docs.claude.com/en/docs/about-claude/models/overview), [OpenRouter](https://openrouter.ai/models), [Azure](https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-models/concepts/models-sold-directly-by-azure). **Note:** For STRUCTURED_OUTPUT format, only certain models support structured outputs. See [OpenAI Structured Outputs Guide](https://platform.openai.com/docs/guides/structured-outputs) for supported models.", + "title": "Instruct model", + "description": "The identifier of the chat model to ask. Some providers do not offer a model listing, in which case type the identifier in directly rather than picking it from the list. Structured output additionally requires a model that implements it; the provider's own model overview says which ones do. Model identifiers for some public providers: [OpenAI](https://platform.openai.com/docs/models), [Claude](https://docs.claude.com/en/docs/about-claude/models/overview), [OpenRouter](https://openrouter.ai/models), [Azure](https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-models/concepts/models-sold-directly-by-azure).", "type": "string", "parameterType": "string", "value": null, @@ -1937,8 +2231,8 @@ }, "developer_prompt_template": { "name": "developer_prompt_template", - "title": "Developer Prompt Template", - "description": "The developer (system) prompt inserted at `{{ developer_prompt }}` in the Messages Template. Defines the LLM's role and behaviour.", + "title": "Developer prompt template", + "description": "The developer (system) prompt, which sets the model's role and behaviour. It is substituted into the messages template wherever `{{ developer_prompt }}` appears, but it is not itself rendered against the entity: Jinja variables written here stay in the prompt as literal text.", "type": "string", "parameterType": "code-jinja2", "value": "You are a helpful assistant.", @@ -1948,8 +2242,8 @@ }, "instruct_prompt_template": { "name": "instruct_prompt_template", - "title": "Instruction Prompt Template", - "description": "The instruction (user) prompt inserted at `{{ instruction_prompt }}` in the Messages Template. Please have a look at the task documentation for detailed instructions.", + "title": "Instruction prompt template", + "description": "The instruction (user) prompt, the actual request made of the model. It is rendered against each entity and the result is substituted into the messages template wherever `{{ instruction_prompt }}` appears. The variables it uses decide whether the task has an input port and what that port asks for, which the task documentation describes in full.", "type": "string", "parameterType": "code-jinja2", "value": "Write a paragraph about this entity: {{ entity }}", @@ -1959,8 +2253,8 @@ }, "enable_file_inputs": { "name": "enable_file_inputs", - "title": "Enable File Inputs", - "description": "Send files from CMEM to the LLM as multimodal input. When enabled, the input port accepts files (FileEntitySchema). Each file is base64-encoded and embedded into the chat message: images (e.g. PNG, JPEG) are sent as image content parts; documents and other files (e.g. PDF, spreadsheets, rich documents, presentations, text and code) are sent as file content parts. Which MIME types are supported is highly dependent on the provider and model used. Entity-derived template variables (e.g. `{{ entity.name }}`) are not available in this mode. The `developer_prompt_template` as well as `instruct_prompt_template` are sent alongside the image/file. For TEXT and JSON_MODE outputs, the output entity carries `_filename` and `_mimetype` paths so downstream tasks can see what the model saw. Note: requires a multimodal-capable model for the chosen file type.", + "title": "Enable file inputs", + "description": "If enabled, the task sends files instead of entity data. The input port then accepts file entities, and each file is base64 encoded into its own request: images go as image content parts and everything else - PDFs, spreadsheets, presentations, text and code - as file content parts, with the filename preserved. A file carrying no MIME type is classified by its extension. The model has to be able to read the type you send it, and which types a provider accepts varies widely. Both prompts are sent alongside the file, but there is no entity left for them to read, so entity variables in the instruction prompt have nothing to render. The messages template is replaced by a built-in one. For text and JSON mode the output additionally carries `_filename` and `_mimetype`, so a later task can see what the model saw; structured output keeps its schema to the declared fields and adds neither.", "type": "string", "parameterType": "boolean", "value": "false", @@ -1973,7 +2267,7 @@ "base_url": { "name": "base_url", "title": "Base URL", - "description": "The base URL of the OpenAI compatible API (without endpoint path). If left empty CMEMs internal LLM proxy is used.", + "description": "The base URL of the OpenAI compatible API, without the endpoint path. Left empty, the task uses the built-in large language model proxy of the eccenca Corporate Memory deployment it runs in. That fallback does not apply to `AZURE_OPENAI`, which always needs a URL here.", "type": "string", "parameterType": "string", "value": "", @@ -1983,8 +2277,8 @@ }, "api_type": { "name": "api_type", - "title": "API Type", - "description": "Select the API client type. This determines the authentication method and endpoint configuration used for API requests. Choose `OPENAI` for direct OpenAI API access or `AZURE_OPENAI` for Azure-hosted OpenAI services. Consider using the API version advanced parameter in case you access Azure-hosted OpenAI services.", + "title": "API type", + "description": "Which client is used to reach the API. `OPENAI` speaks the plain OpenAI protocol and covers OpenAI itself as well as every compatible service. `AZURE_OPENAI` uses Azure's authentication and URL layout, and needs both a base URL and an API version.", "type": "string", "parameterType": "enumeration", "value": "OPENAI", @@ -1995,7 +2289,7 @@ "api_key": { "name": "api_key", "title": "API key", - "description": "An optional API key for authentication. When using CMEMs internal LLM proxy this parameter is ignored and `EXPLORE_AI_APIKEY` in `environments/config.env` is used.", + "description": "The key used to authenticate against the API. It is ignored when the built-in model proxy is used, which authenticates with the `EXPLORE_AI_APIKEY` entry of `environments/config.env` instead.", "type": "string", "parameterType": "password", "value": "", @@ -2005,8 +2299,8 @@ }, "api_version": { "name": "api_version", - "title": "API Version", - "description": "Azure OpenAI API version (only used when API Type is `AZURE_OPENAI`). For more information about OpenAI API version at Azure, please see [the documentation](https://learn.microsoft.com/en-gb/azure/ai-foundry/openai/api-version-lifecycle).", + "title": "API version", + "description": "The Azure OpenAI API version, read only when the API type is `AZURE_OPENAI`. The versions currently offered are listed in [the Azure documentation](https://learn.microsoft.com/en-gb/azure/ai-foundry/openai/api-version-lifecycle).", "type": "string", "parameterType": "string", "value": "", @@ -2016,8 +2310,8 @@ }, "temperature": { "name": "temperature", - "title": "Temperature (between 0 and 2)", - "description": "A parameter that controls the randomness and creativity of the model. A high temperature value (`0.8` - `1.0`) increases randomness and creativity. This is useful for open-ended tasks like storytelling or brainstorming. A low temperature value (`0.0` - `0.4`) produces more deterministic and focused outputs. This is suitable for factual or technical tasks.", + "title": "Temperature", + "description": "How random the model is allowed to be, between `0` and `2`. A high value (`0.8` - `1.0`) makes the output more varied and more creative, which suits open-ended writing. A low value (`0.0` - `0.4`) makes it more repeatable and more focused, which suits factual and technical work.", "type": "string", "parameterType": "double", "value": "1.0", @@ -2028,7 +2322,7 @@ "timeout": { "name": "timeout", "title": "Timeout (seconds)", - "description": "The timeout for a single API request in seconds.", + "description": "How long a single API request may take before it is abandoned, in seconds.", "type": "string", "parameterType": "double", "value": "300", @@ -2038,8 +2332,8 @@ }, "instruction_output_path": { "name": "instruction_output_path", - "title": "Instruction Output Path", - "description": "The entity path where the instruction result will be provided. Note: This parameter is not used when Output Format is set to STRUCTURED_OUTPUT. For structured outputs, only the Pydantic model fields are included in the output schema.", + "title": "Instruction output path", + "description": "The entity path the model's answer is written to. It may not be a path the prompt reads. Structured output does not use it, because the fields of the schema become the output paths instead.", "type": "string", "parameterType": "string", "value": "_instruction_output", @@ -2049,8 +2343,8 @@ }, "messages_template": { "name": "messages_template", - "title": "Messages Template", - "description": "A list of messages comprising the conversation compatible with OpenAI chat completion API message object. Ignored if `enable_file_inputs` is active. Have look at [Message roles and instruction following](https://platform.openai.com/docs/guides/text#message-roles-and-instruction-following) to learn about different levels of priority to messages with different roles.", + "title": "Messages template", + "description": "The list of messages sent to the chat completions API, as JSON. `{{ developer_prompt }}` and `{{ instruction_prompt }}` are replaced by the two prompts, and the template has to mention `instruction_prompt` somewhere. Use it to add further messages, to give an example exchange, or to assign the prompts to roles other than the default ones - [message roles and instruction following](https://platform.openai.com/docs/guides/text#message-roles-and-instruction-following) explains what the roles weigh. It is replaced by a built-in template when file inputs are enabled.", "type": "string", "parameterType": "code-json", "value": "[\n {\n \"role\": \"developer\",\n \"content\": \"{{ developer_prompt }}\"\n },\n {\n \"role\": \"user\",\n \"content\": \"{{ instruction_prompt }}\"\n }\n]", @@ -2060,8 +2354,8 @@ }, "output_format": { "name": "output_format", - "title": "Output Format", - "description": "Specifying the format that the model must output. Possible values are `TEXT` - Standard text output, `STRUCTURED_OUTPUT` - output follows a given schema. Add your schema as Pydantic model in the parameter below, `JSON_MODE` - a more basic version of the structured outputs feature where you have to add your structure to the prompt template.", + "title": "Output format", + "description": "What the model is asked to return. `TEXT` takes the answer as it comes. `JSON_MODE` makes the provider guarantee that the answer parses as JSON, while the structure it should have remains something to describe in the prompt. `STRUCTURED_OUTPUT` hands the provider a schema instead, so the answer comes back in the shape the Pydantic schema gives.", "type": "string", "parameterType": "enumeration", "value": "TEXT", @@ -2071,8 +2365,8 @@ }, "pydantic_schema": { "name": "pydantic_schema", - "title": "Pydantic Schema", - "description": "The Pydantic schema definition with a mandatory class named `StructuredOutput(BaseModel)`. This is only used in combination with the Structured Output format. A schema may have up to 100 object properties total, with up to 5 levels of nesting. The total string length of all property names, definition names, enum values, and const values cannot exceed 15,000 characters.", + "title": "Pydantic schema", + "description": "The schema the answer has to follow, used by the structured output format. It is a Pydantic model, and the root class has to be named `StructuredOutput`. Only class definitions with typed fields are accepted here, next to imports from `pydantic`, `typing`, `typing_extensions`, `datetime` and `enum`. Methods, plain assignments and anything else executable are rejected. Nested models are allowed and become sub-entities on the output. Providers place their own limits on the schema, commonly at 100 properties, five levels of nesting, and 15,000 characters across all property, definition, enum and constant names.", "type": "string", "parameterType": "code-python", "value": "from pydantic import BaseModel\n\nclass StructuredOutput(BaseModel):\n title: str\n abstract: str\n keywords: list[str]\n", @@ -2083,7 +2377,7 @@ "raise_on_error": { "name": "raise_on_error", "title": "Raise on API errors", - "description": "How to react on API errors. When enable, any API errors will cause the workflow to stop with an exception. When disabled, API errors are logged and the error message is written to the entity output, allowing the workflow to continue processing other entities.", + "description": "If enabled, an error reported by the API stops the workflow. If disabled, the error is logged, its text is written to the output in place of an answer, and the remaining entities are still processed. Errors that are not the API's - an unrenderable prompt, an unreadable file - stop the workflow either way.", "type": "string", "parameterType": "boolean", "value": "true", @@ -2093,8 +2387,8 @@ }, "max_concurrent_requests": { "name": "max_concurrent_requests", - "title": "Maximum Concurrent Requests", - "description": "Maximum number of concurrent API requests to prevent rate limiting and resource exhaustion.", + "title": "Maximum concurrent requests", + "description": "How many API requests may be in flight at once. This is what governs how fast a run gets through its entities, and lowering it is the first answer to a provider that rate-limits.", "type": "string", "parameterType": "Long", "value": "10", @@ -2104,8 +2398,8 @@ }, "batch_size": { "name": "batch_size", - "title": "Batch Size", - "description": "Number of entities to process in each batch for memory optimization.", + "title": "Batch size", + "description": "How many entities are prepared and processed together. Entities are read and handed on one batch at a time rather than all at once, so a smaller batch keeps less in memory and lets results reach the next task sooner.", "type": "string", "parameterType": "Long", "value": "100", @@ -2115,8 +2409,8 @@ }, "request_delay": { "name": "request_delay", - "title": "Request Delay (seconds)", - "description": "Delay between API requests in seconds to respect rate limits.", + "title": "Request delay (seconds)", + "description": "How long to wait before each API request, in seconds. Left at zero, requests are sent as fast as the concurrency limit allows. Raise it for a provider that rejects bursts even within its request limit.", "type": "string", "parameterType": "double", "value": "0.0", @@ -2325,8 +2619,8 @@ "Uncategorized" ], "main_category": "Uncategorized", - "description": "REST operator that fetches and optionally merges data from a REST endpoint. It supports executing multiple requests either via input entities that each overwrite config parameters or via paging. If you only need to download a single file, the \"Download file\" operator might be the better option. Most features are currently only supported for JSON REST APIs. From multiple requests the REST operator can produce a merged JSON result, i.e. for JSON it will concatenate all results in a JSON array. Alternatively multiple results can be written directly to file (of a JSON dataset), either as a merged JSON file or one file per request inside a ZIP file. By default the output of this operator is an entity with a single property 'result', which is the (concatenated) JSON string.", - "markdownDocumentation": "## Core parameter overview\n\n- `URL`: The URL the request will be executed against. This value can be overwritten at execution time when the 'Read parameters from input' option\n is enabled. This value will also be adapted when a paging approach is configured, see the paging section for more details. \n- `Method`: One of the following HTTP methods: GET, POST, PUT, PATCH or DELETE.\n- `Accept`: The ACCEPT header value for content negotiation, e.g. 'application/json'.\n- `Content type`: The CONTENT-TYPE header value. This is usually used for POST, PUT or PATCH requests when the API endpoint\n supports multiple different MIME types and/or requires a content MIME type to be set. E.g. 'application/json'\n- `Content`: The text content of a POST, PUT or PATCH request. This value can be overwritten at execution time when the\n 'Read parameters from input' option is enabled.\n\n## Authorization\n\nIf the request needs authorization following parameters should be set, else there is no authorization header sent.\n\n- `Authorization header`: The header that is used for authorization, usually either 'Authorization' or 'Proxy-Authorization'.\n- `Authorization header value`: The secret value for the authorization, i.e. password or token. This value will be encrypted\n and cannot be accessed in the user interface anymore after saving it.\n E.g. for OAuth the value would have the following form: `bearer `.\n\n## Sending multiple requests\n\nIn the default configuration a single requests is sent. If multiple requests should be sent with different URLs and/or content,\nthe configurations for these requests must be defined via the input port of the REST operator.\n\n- Read parameters from input:\n The 'URL' and 'Content' parameter values are read from entities that are input via the input\n port of the operator. The property names are 'url' and 'content' and only overwrite the\n original parameter value if defined.\n\n For each input entity a separate request will be sent.\n- Limit: If set to a positive number, then only that number of input entities will be processed as requests.\n- Offset: If set to a positive number, then that many input entities will be ignored before processing them as requests.\n\nIf the option 'Read parameters from input' is enabled, it is currently always assumed that multiple requests will be sent.\nThe responses must either be of type JSON, then the results are merged into a JSON array, or the 'Output result as file'\noption must be enabled in order to write a merged JSON or a ZIP file. See section 'Output options' for more details.\n\n## Paging\n\nIf the REST endpoint does not return all results in a single response, multiple requests (one per page) must usually be sent in order\nto fetch all results. This is currently only supported for JSON requests.\n\n- `Paging method`:\n There are two paging methods currently supported:\n\n 1. `Next page full URL`: The JSON response contains the full URL of the next page. This URL will be used for the subsequent request URL.\n 2. `Next page identifier`: The JSON response contains the ID of the next page. This ID will be used as query parameter value for the subsequent request.\n\n In both cases the path to the next page value in the response JSON must be defined via the 'Next page JSON path' parameter.\n In case of the 'Next page identifier' paging method, also the parameter 'Next page ID query parameter' must be set.\n- `Next page JSON path`: The property path in the result JSON where the 'next page' URL/value is provided.\n E.g. for following response structure, the value for this parameter would be `paging/next`:\n\n ```\n {\n ...,\n \"paging\": {\n \"next\": \"Next ID\"\n }\n }\n ```\n- `Next page ID query parameter`: If the paging method is 'Next page identifier', this defines the query parameter name that should\n be attached to the original request URL in combination with the 'next page' value of the current response in order\n to request the next page.\n\n## Setting HTTP headers\n\n- `HTTP headers`: This parameter allows to set HTTP headers of the request being made. Each line of the multi-line value should contain a single header, e.g.\n ```\n Accept-Language: en-US,en;q=0.5\n Cache-Control: max-age=0\n ```\n \n## Sending a multipart HTTP file request\n\nIf the content of a POST request should be sent as file content of a multipart HTTP request, instead of the request body,\nfollowing parameter must be configured:\n\n- `Multi-part file parameter`: If set to a non-empty value then, instead of a normal POST request, a multipart/form-data\n file upload request will be executed.\n The value of this parameter is used as the form parameter name.\n \n## Output options\n\nBy default, the response body of a request is output as value of the 'result' property of a single output entity.\nIf the response body needs to be processed this can e.g. be achieved with the 'Parse JSON' operator. Alternatively\nthe response/s can be written to a file based dataset. Currently only text based datasets are supported.\n\nThe results of multiple requests, see section 'Sending multiple requests' for details, can be written to\na single, merged file (only supported for JSON) or to a ZIP archive, i.e. a file resource that must end in '.zip'.\nIn the latter case an entry per request is added to the ZIP file.\nCurrently, the following datasets support the processing of ZIP files: JSON, XML, CSV and RDF file.\n\n- `Output result as file`: If enabled, instead of outputting a single entity, the result/s will be written directly \n to the file of the file-based dataset that is connected to the output of this operator.\n\nIf the option 'Read parameters from input' is enabled, it is currently always assumed that multiple requests will be sent.\nThe responses must either be JSON, then the results are merged into a JSON array or the 'Output result as file'\noption must be enabled in order to write a merged JSON or ZIP file.\n \n## Fine-tuning timeouts\n\nIf requests can take a much longer time than what can usually be expected, it is possible to increase the timeouts to\ncontrol when a request should eventually fail.\n\n- `Request timeout`: The maximum overall time in milliseconds the request is allowed to take. Default: `10000`.\n- `Connection timeout`: The maximum time in milliseconds the request is allowed to establish a connection to the server. Default: `5000`.\n- `Read timeout`: The maximum time a request is allowed to stay idle, i.e. the time while it receives no data. Usually this\n should be greater than the time span between the request being sent and the first data being received. Default: `10000`\n\n## Throttling requests\n\nIf a lot of requests are sent via the 'Read parameters from input' option, it can make sense to throttle the number\nof requests sent in a specific time span.\n\n- `Delay between requests`: The delay between subsequent requests in milliseconds. Default: `0`.\n\n## Error handling\n\nFollowing parameters can be tuned in order to decide when an execution should be considered as failed.\n\n- `Retries per request`: How often a single request configuration (URL, content) should be retried before considering this\n request configuration as failed. Default: `3`\n- `Abort when request fails`: When enabled, if a single request configuration eventually fails, i.e. it reaches its max. retry count,\n the overall execution of the REST operator will fail.\n- `Max failed requests`: If set to a value greater 0, the execution will abort if more than the given number of request configurations \n have failed (reached max. retries). This can be used if a number of failed requests can be tolerated.\n When 'Abort when request fails' is enabled, this option is ignored.\n\n## Propagating the request URL\n\nIf having the request URL in the response data is needed, following parameter needs to be configured:\n\n- `URL property`: If this parameter is non-empty the request URL will be added to the response JSON object. It will be added as value to\n a property with the specified name in the root level of the response JSON object.\n This is mostly relevant if the request URL cannot be re-constructed from the response data. Only supported for JSON responses.\n", + "description": "REST operator that fetches and optionally merges data from a REST endpoint. It supports executing multiple requests either via input entities that each overwrite config parameters or via paging. If you only need to download a single file, the \"Download file\" operator might be the better option. Most features are currently only supported for JSON REST APIs. By default the output of this operator is one entity per response with a single property 'result' that holds the response body. Responses of multiple requests are merged into a single JSON array if the 'Parse JSON' operator is connected or the Accept header starts with 'application/json'. Alternatively the response/s can be written to the file of a connected file-based dataset, either as a merged JSON file (JSON dataset) or, if the dataset file ends in '.zip', one file per response inside a ZIP file.", + "markdownDocumentation": "## Core parameter overview\n\n- `URL`: The URL the request will be executed against. This value can be overwritten at execution time when the 'Read parameters from input' option\n is enabled. This value will also be adapted when a paging approach is configured, see the paging section for more details. \n- `Method`: One of the following HTTP methods: GET, POST, PUT, PATCH or DELETE.\n- `Accept`: The ACCEPT header value for content negotiation, e.g. 'application/json'.\n- `Content type`: The CONTENT-TYPE header value. This is usually used for POST, PUT or PATCH requests when the API endpoint\n supports multiple different MIME types and/or requires a content MIME type to be set. E.g. 'application/json'\n- `Content`: The text content of a POST, PUT or PATCH request. This value can be overwritten at execution time when the\n 'Read parameters from input' option is enabled.\n\n## Authorization\n\nIf the request needs authorization following parameters should be set, else there is no authorization header sent.\n\n- `Authorization header`: The header that is used for authorization, usually either 'Authorization' or 'Proxy-Authorization'.\n- `Authorization header value`: The secret value for the authorization, i.e. password or token. This value will be encrypted\n and cannot be accessed in the user interface anymore after saving it.\n E.g. for OAuth the value would have the following form: `bearer `.\n\n## Secrets in the request content and headers\n\nTemplates of the `Content` and `HTTP headers` parameters may reference sensitive variables, e.g. `secret={{global.apiSecret}}`\nas content or `X-Api-Key: {{global.apiKey}}` as a header line. The evaluated value is sent with the request but not displayed:\nthe template preview, the task properties and the search show the placeholder `\u2022\u2022\u2022\u2022\u2022\u2022` instead.\nAs with password parameters, anyone who can edit the task can send the value to any host.\n\n## Sending multiple requests\n\nIn the default configuration a single requests is sent. If multiple requests should be sent with different URLs and/or content,\nthe configurations for these requests must be defined via the input port of the REST operator.\n\n- Read parameters from input:\n The 'URL' and 'Content' parameter values are read from entities that are input via the input\n port of the operator. The property names are 'url' and 'content' and only overwrite the\n original parameter value if defined.\n\n For each input entity a separate request will be sent.\n- Limit: If set to a positive number, then only that number of input entities will be processed as requests.\n- Offset: If set to a positive number, then that many input entities will be ignored before processing them as requests.\n\nIf the option 'Read parameters from input' is enabled or a paging method is configured, it is always assumed that multiple\nrequests will be sent. See section 'Output options' for how the responses of multiple requests are output.\n\n## Paging\n\nIf the REST endpoint does not return all results in a single response, multiple requests (one per page) must usually be sent in order\nto fetch all results. This is currently only supported for JSON requests.\n\n- `Paging method`:\n There are two paging methods currently supported:\n\n 1. `Next page full URL`: The JSON response contains the full URL of the next page. This URL will be used for the subsequent request URL.\n 2. `Next page identifier`: The JSON response contains the ID of the next page. This ID will be used as query parameter value for the subsequent request.\n\n In both cases the path to the next page value in the response JSON must be defined via the 'Next page JSON path' parameter.\n In case of the 'Next page identifier' paging method, also the parameter 'Next page ID query parameter' must be set.\n- `Next page JSON path`: The property path in the result JSON where the 'next page' URL/value is provided.\n E.g. for following response structure, the value for this parameter would be `paging/next`:\n\n ```\n {\n ...,\n \"paging\": {\n \"next\": \"Next ID\"\n }\n }\n ```\n- `Next page ID query parameter`: If the paging method is 'Next page identifier', this defines the query parameter name that should\n be attached to the original request URL in combination with the 'next page' value of the current response in order\n to request the next page.\n\n## Setting HTTP headers\n\n- `HTTP headers`: This parameter allows to set HTTP headers of the request being made. Each line of the multi-line value should contain a single header, e.g.\n ```\n Accept-Language: en-US,en;q=0.5\n Cache-Control: max-age=0\n ```\n \n## Sending a multipart HTTP file request\n\nIf the content of a POST request should be sent as file content of a multipart HTTP request, instead of the request body,\nfollowing parameter must be configured:\n\n- `Multi-part file parameter`: If set to a non-empty value then, instead of a normal POST request, a multipart/form-data\n file upload request will be executed.\n The value of this parameter is used as the form parameter name.\n \n## Output options\n\nBy default, the response body of a request is output as value of the 'result' property of an output entity.\nIf the response body needs to be processed this can e.g. be achieved with the 'Parse JSON' operator.\nIf multiple requests are sent, see sections 'Sending multiple requests' and 'Paging', one entity per response is output.\nOnly if the 'Parse JSON' operator is connected to the output or the 'Accept' parameter starts with 'application/json',\nthe responses are merged into a single JSON array that is output as a single entity. In that case each response of a\nGET request must be valid JSON, otherwise the request is considered failed. Responses to other HTTP methods that are not\nvalid JSON are skipped.\n\nAlternatively, the response/s can be written to the file of a file-based dataset:\n\n- `Output result as file`: If enabled, instead of outputting entities, the result/s will be written directly\n to the file of the file-based dataset that is connected to the output of this operator.\n\nWith this option enabled, the operator outputs a file instead of entities. The task connected to the output must be a\nfile-based dataset, e.g. JSON, XML, CSV, Text or RDF file. The file of that dataset is replaced by the written file,\nso its previous content is lost. Afterwards the responses can be processed like any other file of the dataset, e.g. by\na transformation that reads the dataset. Currently only text based responses are supported. Responses with HTTP status\n204 (No Content) are not written.\n\n### Single request\n\nIf a single request is sent, the response body is written to the dataset file as is. If the dataset file is a ZIP archive,\ni.e. its name ends in '.zip', the response is written as the only entry of the archive, named as described under\n'Multiple requests'. If the dataset is a JSON dataset with a file that does not end in '.zip', the response of a GET request\nmust be valid JSON, otherwise the request is considered failed, and the 'URL property' option, see section 'Propagating\nthe request URL', is applied to the valid response before it is written.\n\n### Multiple requests\n\nIf multiple requests are sent, all responses are written to a single file in one of the following ways:\n\n- ZIP archive: If the file of the connected dataset ends in '.zip', one entry per response is added to the archive.\n This applies to all file-based datasets, including a JSON dataset, for which no merged JSON array is written in that case.\n The entries are named 'responseBody' followed by a ten-digit counter, e.g. `responseBody0000000001.json`. The suffix\n is '.json', '.csv' or '.xml' for the JSON, CSV, Multi CSV ZIP and XML datasets and omitted for other datasets.\n The following datasets can read ZIP archives: JSON, XML, CSV, Multi CSV ZIP, Text, Binary file and RDF file. An RDF file dataset\n needs its 'format' parameter set, since the format is guessed from the dataset file name, which ends in '.zip'.\n- Merged JSON file: If a JSON dataset with a file that does not end in '.zip' is connected, the responses are merged into\n a JSON array, i.e. `[,,...]`. Each response of a GET request must be valid JSON, otherwise the\n request is considered failed. For other HTTP methods, responses that are not valid JSON are skipped. The 'URL property'\n option is applied to each response before it is written.\n\nWriting multiple responses to a dataset that is neither a JSON dataset nor based on a ZIP file fails with a validation error.\n\n## Fine-tuning timeouts\n\nIf requests can take a much longer time than what can usually be expected, it is possible to increase the timeouts to\ncontrol when a request should eventually fail.\n\n- `Request timeout`: The maximum overall time in milliseconds the request is allowed to take. Default: `10000`.\n- `Connection timeout`: The maximum time in milliseconds the request is allowed to establish a connection to the server. Default: `5000`.\n- `Read timeout`: The maximum time a request is allowed to stay idle, i.e. the time while it receives no data. Usually this\n should be greater than the time span between the request being sent and the first data being received. Default: `10000`\n\n## Throttling requests\n\nIf a lot of requests are sent via the 'Read parameters from input' option, it can make sense to throttle the number\nof requests sent in a specific time span.\n\n- `Delay between requests`: The delay between subsequent requests in milliseconds. Default: `0`.\n\n## Error handling\n\nFollowing parameters can be tuned in order to decide when an execution should be considered as failed.\n\n- `Retries per request`: How often a single request configuration (URL, content) should be retried before considering this\n request configuration as failed. Default: `3`\n- `Abort when request fails`: When enabled, if a single request configuration eventually fails, i.e. it reaches its max. retry count,\n the overall execution of the REST operator will fail.\n- `Max failed requests`: If set to a value greater 0, the execution will abort if more than the given number of request configurations \n have failed (reached max. retries). This can be used if a number of failed requests can be tolerated.\n When 'Abort when request fails' is enabled, this option is ignored.\n\n## Propagating the request URL\n\nIf having the request URL in the response data is needed, following parameter needs to be configured:\n\n- `URL property`: If this parameter is non-empty the request URL will be added to the response JSON object. It will be added as value to\n a property with the specified name in the root level of the response JSON object.\n This is mostly relevant if the request URL cannot be re-constructed from the response data. Only supported for JSON responses.\n", "pluginIcon": "data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiPz4KPHN2ZyB3aWR0aD0iMzJweCIgaGVpZ2h0PSIzMnB4IiBmaWxsPSJjdXJyZW50Q29sb3IiIHZpZXdCb3g9IjAgMCAzMiAzMiIgdmVyc2lvbj0iMS4xIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPgogICAgPHRpdGxlPkV4ZWN1dGUgUkVTVCByZXF1ZXN0czwvdGl0bGU+CiAgICA8ZyBpZD0iRXhlY3V0ZS1SRVNULXJlcXVlc3RzIj4KICAgICAgICA8cGF0aCBkPSJNMTcsMjguMDAwMzQ0MyBMNSwyOC4wMDAzNDQzIEw1LDQuMDAwMzQ0MjkgTDE1LDQuMDAwMzQ0MjkgTDE1LDEwLjAwMDM0NDMgQzE1LjAwMzE4NTYsMTEuMTAzNTkxNiAxNS44OTY3NTI3LDExLjk5NzE1ODcgMTcsMTIuMDAwMzQ0MyBMMjMsMTIuMDAwMzQ0MyBMMjMsMTUuMDAwMzQ0MyBMMjUsMTUuMDAwMzQ0MyBMMjUsMTAuMDAwMzQ0MyBDMjUuMDA3MjUwMiw5LjczNDM1MzI0IDI0Ljg5NzYxNDQsOS40Nzg1MzYyNCAyNC43LDkuMzAwMzQ0MjkgTDE3LjcsMi4zMDAzNDQyOSBDMTcuNTIxODQ4MywyLjEwMjY3MzY5IDE3LjI2NjAwNCwxLjk5MzAyNjE1IDE3LDIuMDAwMzQ0MjkgTDUsMi4wMDAzNDQyOSBDMy44OTY3NTI3LDIuMDAzNTI5ODcgMy4wMDMxODU1OCwyLjg5NzA5NyAzLDQuMDAwMzQ0MjkgTDMsMjguMDAwMzQ0MyBDMy4wMDMxODU1OCwyOS4xMDM1OTE2IDMuODk2NzUyNywyOS45OTcxNTg3IDUsMzAuMDAwMzQ0MyBMMTcsMzAuMDAwMzQ0MyBMMTcsMjguMDAwMzQ0MyBaIE0xNyw0LjQwMDM0NDI5IEwyMi42LDEwLjAwMDM0NDMgTDE3LDEwLjAwMDM0NDMgTDE3LDQuNDAwMzQ0MjkgWiIgaWQ9IlNoYXBlIj48L3BhdGg+CiAgICAgICAgPHBvbHlnb24gaWQ9IlBhdGgiIHBvaW50cz0iMTAuODMgMTkgMTMuNDEgMTYuNDIgMTIgMTUgOCAxOSAxMiAyMyAxMy40MiAyMS41OSI+PC9wb2x5Z29uPgogICAgICAgIDxwb2x5Z29uIGlkPSJQYXRoIiBwb2ludHM9IjE3LjU5IDE5IDE1LjAxIDIxLjU4IDE2LjQyIDIzIDIwLjQyIDE5IDE2LjQyIDE1IDE1IDE2LjQxIj48L3BvbHlnb24+CiAgICAgICAgPHBvbHlnb24gaWQ9IlBhdGgiIHBvaW50cz0iMzAgMjUgMjUuMzg0NjE1NCAzMCAyNC4zMDc2OTIzIDI4LjgzMzMzMzMgMjcuMDc2OTIzMSAyNS44MzMzMzMzIDIwIDI1LjgzMzMzMzMgMjAgMjQuMTY2NjY2NyAyNy4wNzY5MjMxIDI0LjE2NjY2NjcgMjQuMzA3NjkyMyAyMS4xNjY2NjY3IDI1LjM4NDYxNTQgMjAiPjwvcG9seWdvbj4KICAgIDwvZz4KPC9zdmc+", "properties": { "url": { @@ -2409,7 +2703,7 @@ "content": { "name": "content", "title": "Content", - "description": "The content that is send with a POST, PUT or PATCH request. For handling this payload dynamically this parameter must be overwritten via the task input.", + "description": "The content that is send with a POST, PUT or PATCH request. For handling this payload dynamically this parameter must be overwritten via the task input. A template of this parameter may use sensitive variables; their values are sent with the request but not displayed.", "type": "string", "parameterType": "string", "value": "", @@ -2420,7 +2714,7 @@ "httpHeaders": { "name": "httpHeaders", "title": "HTTP headers", - "description": "Configure additional HTTP headers. One header per line. Each header entry follows the curl syntax.", + "description": "Configure additional HTTP headers. One header per line. Each header entry follows the curl syntax. A template of this parameter may use sensitive variables; their values are sent with the request but not displayed.", "type": "string", "parameterType": "multiline string", "value": "", @@ -2574,7 +2868,7 @@ "outputResultAsFile": { "name": "outputResultAsFile", "title": "Output result as file", - "description": "If a file based dataset is connected to the output of the REST operator, then this option can be enabled in order to overwrite the file resource of the connected dataset. This allows for handling the result of the REST request/s as a normal dataset. If a non-file based dataset is connected to this operator the execution will fail. If disabled, a single entity with a single property 'result' will be output that contains the (merged) result.", + "description": "If enabled, the response/s are written to the file of the file-based dataset connected to the output, replacing its previous content. A single response is written as is, or as the only entry of a ZIP file. Multiple responses are written as one entry per response into a ZIP file if the dataset file ends in '.zip', or merged into a JSON array if a JSON dataset is connected. The task connected to the output must be a file-based dataset. If disabled, the response/s are output as entities with a single property 'result'.", "type": "string", "parameterType": "boolean", "value": "false", @@ -3021,13 +3315,13 @@ ], "main_category": "Uncategorized", "description": "Generate SHACL node and property shapes from a data graph", - "markdownDocumentation": "This workflow task generates SHACL (Shapes Constraint Language)\nnode and property shapes by analyzing instance data from a knowledge graph. The generated\nshapes describe the structure and properties of the classes used in the data graph.\n\n## Usage\n\nThe plugin analyzes an input data graph and creates:\n\n- **Node shapes**: One for each class (`rdf:type`) used in the data graph\n- **Property shapes**: For all properties associated with each class, including:\n - Regular object properties (subject \u2192 object relationships)\n - Inverse object properties (object \u2190 subject relationships, marked with \u2190 prefix)\n - Datatype properties (literal values)\n\n## Output\n\nThe generated shapes are written to a shape catalog graph with:\n\n- Unique URIs based on UUIDs (UUID5 derived from class/property IRIs)\n- Human-readable labels and names (using namespace prefixes when available)\n- Metadata including source data graph reference and timestamps\n- Optional plugin provenance information (see advanced options)\n\n## Example\n\nGiven a data graph with:\n\n``` turtle\nex:Person123 a ex:Person ;\n ex:name \"John\" ;\n ex:knows ex:Person456 .\n```\n\nThe plugin generates:\n\n- A node shape for `ex:Person` with `sh:targetClass ex:Person`\n\n``` turtle\ngraph:90ee6e27-59b1-5ac8-9d7a-116c60c6791a a sh:NodeShape ;\n rdfs:label \"Person (ex:)\"@en ;\n sh:name \"Person (ex:)\"@en ;\n sh:property\n graph:0fcf371d-f99a-5eeb-ab50-6e6b5fbb0e06 ,\n graph:dd5c6728-75a2-5215-8a5d-f9cd4077aaea ;\n sh:targetClass ex:Person .\n```\n\n- Property shapes for `ex:name` (datatype property) and `ex:knows` (object property)\n\n``` turtle\ngraph:0fcf371d-f99a-5eeb-ab50-6e6b5fbb0e06 a sh:PropertyShape ;\n rdfs:label \"knows (ex:)\"@en ;\n sh:name \"knows (ex:)\"@en ;\n sh:nodeKind sh:IRI ;\n sh:path ex:knows ;\n shui:showAlways true .\n```\n\n", + "markdownDocumentation": "This workflow task generates SHACL (Shapes Constraint Language) node\nand property shapes by analyzing the instance data of a knowledge graph. The generated\nshapes describe the classes the data uses and the properties each class is used with.\n\nThe data graph and the shape catalog are both chosen by IRI parameter rather than by\nworkflow connection, so the task has neither an input nor an output port. It is a\nterminal step: it reads and writes graphs directly and hands nothing on to a following\ntask.\n\n## What is generated\n\n- A **node shape** for every class that instances are typed with.\n- A **property shape** for every property those instances use: object properties in the\n subject \u2192 object direction, object properties in the object \u2190 subject direction, whose\n names carry a `\u2190` prefix, and datatype properties, whose values are literals.\n\nEvery shape gets an IRI derived from a UUID5 of the class or property it describes, so\nwithin one catalog the same class or property always maps to the same shape. Names and\nlabels come from the title eccenca Corporate Memory resolves for that class or property.\nA description follows wherever one can be resolved - the `rdfs:comment`,\n`dcterms:description` or `skos:definition` of the class or property, looked up wherever\nit is defined, so a vocabulary graph counts as well as the data graph. A class or\nproperty that nothing describes gets no `sh:description`, and neither does a shape for\nthe object \u2190 subject direction: the description is written about the property, so on the\ninverse path it would say the opposite of what the shape holds.\n\nEvery property shape carries `shui:showAlways`, and a shape for the object \u2190 subject\ndirection carries `shui:inversePath` as well. A node shape carries the `foaf:depiction` of\nits target class wherever the data graph or the class's own vocabulary offers one. The\ncatalog itself records the data graph it was generated from, when it was written, and the\nclasses it manages.\n\n## Caveats\n\nNames, labels and descriptions are requested in English. Where a vocabulary offers no\nEnglish text, Corporate Memory answers in whatever language it does have, and the\nliteral carries that language rather than claiming to be English. A class or property\nthe deployment knows nothing about falls back to a name built from its IRI, which has\nno language at all and is written without a tag.\n\nA property used by several classes gets one property shape, shared by every node shape\nthat uses it. Its `sh:nodeKind` is decided by whichever of those classes sorts first, so a\nproperty carrying IRI values under one class and literal values under another is described\nas only one of the two - the same one on every run over the same data.\n\nA class or property that is a blank node is skipped. Instance data sharing a graph with an\nontology types things with anonymous class expressions, and a shape has nothing to target\nin one.\n\n`sh:datatype rdf:langString` is added as soon as any value of a property carries a\nlanguage tag, however few of them do. A property whose values mix tagged and untagged\nliterals therefore gets a shape that its own source data does not satisfy.\n\nAdding to an existing catalog replaces the name, label and description of every shape the\nrun writes, and the classes the catalog declares, rather than leaving an earlier run's\nbeside them. Everything else in the catalog is left alone, including shapes this run did\nnot produce - that is what adding is for.\n\n## Example\n\nGiven a data graph typing instances with a vocabulary the deployment knows:\n\n``` turtle\nex:Person123 a ex:Person ;\n ex:name \"John\" ;\n ex:knows ex:Person456 .\n```\n\nthe task generates a node shape for `ex:Person`\n\n``` turtle\ngraph:90ee6e27-59b1-5ac8-9d7a-116c60c6791a a sh:NodeShape ;\n rdfs:label \"Person (ex:)\"@en ;\n sh:name \"Person (ex:)\"@en ;\n sh:description \"A person.\"@en ;\n sh:property\n graph:0fcf371d-f99a-5eeb-ab50-6e6b5fbb0e06 ,\n graph:dd5c6728-75a2-5215-8a5d-f9cd4077aaea ;\n sh:targetClass ex:Person .\n```\n\ntogether with a property shape for each of `ex:knows` and `ex:name`, of which the first\nreads\n\n``` turtle\ngraph:0fcf371d-f99a-5eeb-ab50-6e6b5fbb0e06 a sh:PropertyShape ;\n rdfs:label \"knows (ex:)\"@en ;\n sh:name \"knows (ex:)\"@en ;\n sh:nodeKind sh:IRI ;\n sh:path ex:knows ;\n shui:showAlways true .\n```\n\n", "pluginIcon": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9Ii01NSAtMjAgNTk3LjMzNSA1OTcuMzM1Ij4NCiAgICA8bWFzayBpZD0ibSI+DQogICAgICAgIDxyZWN0IHk9IjQwLjQ3MyIgd2lkdGg9IjQ4NC44NjIiIGhlaWdodD0iNDg0Ljg2MiIgZmlsbD0id2hpdGUiLz4NCiAgICAgICAgPHJlY3QgeT0iNTAuODA4IiB4PSI5LjQ3NyIgZmlsbD0iYmxhY2siIHdpZHRoPSI0NjQuMTkyIiBoZWlnaHQ9IjMyOS44NDciLz4NCiAgICAgICAgPHRleHQgdHJhbnNmb3JtPSJzY2FsZSgwLjkzLDEpIiB5PSI0OTgiIHg9IjQ3IiBmb250LWZhbWlseT0iVmVyZGFuYSIgZm9udC1zaXplPSIxMTQiIGZvbnQtd2VpZ2h0PSJib2xkIiBmaWxsPSJibGFjayI+U0g8dHNwYW4ga2VybmluZz0iLTMiPkE8L3RzcGFuPkNMPC90ZXh0Pg0KICAgICAgICA8cGF0aCBkPSJNMzgzLjQ3NiwyNjcuMzQzYy0yLjU0NC0xLjM0Ni01LjE0LTIuNDkzLTcuNzQzLTMuNTE2bDEuODYzLTAuMTVjMCwwLTE2LjYwOC03LjM1NC0xOC4wNTctNjAuNzIyICBjLTEuNDM4LTUzLjM3MiwxNS44MjgtNjIuNDc4LDE1LjgyOC02Mi40NzhsLTIuNDgsMC4xMDljMTMuMDQ1LTYuNjksMjQuMjY1LTE3LjI2NywzMS42NjktMzEuMjE2ICBjMTkuMjk1LTM2LjI5MSw1LjQ4OC04MS4zNjItMzAuODEtMTAwLjY1N0MzMzcuNDM2LTEwLjU2MywyOTIuMzc0LDMuMjA3LDI3My4wOSwzOS41M2MtNy45MjcsMTQuODk5LTEwLjE3OCwzMS4yNzMtNy42NzcsNDYuNzMzICBsLTAuODUxLTEuMzA2YzAsMCw0LjM3MywxOS4zNjUtNDEuMDMyLDQ3LjU1Yy00NS4zOTcsMjguMi02NS44NzcsMTQuMTU5LTY1Ljg3NywxNC4xNTlsMS4zMDIsMS45MjUgIGMtMS4yOTgtMC44MDMtMi41NDQtMS42MjQtMy45MDEtMi4zMzNjLTM2LjMwNi0xOS4yOTQtODEuMzgtNS41MDktMTAwLjY2NywzMC44MDRjLTE5LjI4MSwzNi4zMDktNS40ODksODEuMzY1LDMwLjgxMywxMDAuNjY4ICBjMjcuMDY0LDE0LjM2NCw1OC45NzQsMTAuMzYsODEuNDYxLTcuNjU1bC0wLjQ4NywwLjk0NmMwLDAsMTYuNTMxLTEzLjU5OSw2NC4xNiwxMS45NzNjMzcuNjAxLDIwLjE3OCw0My4xODQsMzkuOTU2LDQzLjg5OSw0Ny4zODMgIGMtMC45ODMsMjcuNTcsMTMuMzg4LDU0LjYxOCwzOS4zODksNjguNDMzYzM2LjMwMSwxOS4yOTksODEuMzc0LDUuNDk4LDEwMC42NTctMzAuODA0ICBDNDMzLjU3MSwzMzEuNzA0LDQxOS43ODYsMjg2LjYyNCwzODMuNDc2LDI2Ny4zNDN6IE0yOTkuNTQyLDI3Ny4xMjhjLTYuMDE4LDIuMTI5LTIzLjIwMyw0LjQ4Ny01OS4zODktMTQuOTIxICBjLTM5LjE4Ny0yMS4wNC00NS4wMDUtMzguNjE1LTQ1Ljg1NS00My44OTFjMC41NTctNi40MDEsMC4yMDItMTIuNzkxLTAuODkxLTE5LjAybDAuMjM5LDAuMzU5YzAsMC0zLjE4OS0xNy4wOTYsNDEuNjUtNDQuOTQzICBjNDAuMTMzLTI0LjkwOCw1OC4zNzYtMTkuOTU1LDYxLjc3MS0xOC42NTNjMi4xODUsMS40ODUsNC40NSwyLjg2Nyw2LjgyNSw0LjEzMWM0LjUxOCwyLjM5OCw5LjE3NCw0LjI4MywxMy44ODgsNS42NzIgIGM1LjUyLDUuMjU3LDE1LjY3OCwyMC4xNzgsMTYuNzMzLDU5LjQxM2MxLjA3OCwzOS41MzUtMTAuNTMzLDU0Ljc3OS0xNi44NjUsNjAuMTY4QzMxMS4xMjIsMjY4LjM5OSwzMDUuMDIyLDI3Mi4zNCwyOTkuNTQyLDI3Ny4xMjggIHoiIGZpbGw9IndoaXRlIiBzdHJva2U9ImJsYWNrIi8+DQogICAgICAgIDxwYXRoIGQ9Ik0yOTQuNjMxLDMyLjRjLTIxLjY1OSwyMy42OTQtMjIuMTE4LDU4LjU0NC0xLjI0Niw3OC4yNjRjLTEwLjMxNi05LjkzNS0xMC4wOTMtMzAuNjA4LDAuMzMtNTAuMzg3ICAgYzEuMzQyLTEuNzc0LDUuMjI0LTUuOTYxLDEwLjg5OC00LjA1NWMwLjU2OSwwLjE5NCwwLjk1MSwwLjI0NSwxLjE3LDAuMjEzYzEuMjg0LDAuMjc4LDIuNjAxLDAuNDM5LDMuOTYsMC4zODEgICBjOC42MjItMC4zOTMsMTUuMjg4LTcuNjg0LDE0Ljg5OC0xNi4zMDdjLTAuMTc1LTMuODY4LTEuNzgtNy4yOTItNC4yNi05LjkyYzIwLjAwNy0xMy4xMDcsNDIuOTY4LTE0LjYxLDUyLjM3NS02LjAxNmwwLjM1OSwwLjAyNSAgIEMzNTEuNjI1LDQuOTcsMzE2LjQ5Myw4LjQ3NiwyOTQuNjMxLDMyLjR6IiBmaWxsPSJibGFjayIvPg0KICAgICAgICA8cGF0aCBkPSJNNzYuODk0LDE3Mi41MmMtMjEuNjYzLDIzLjY5NC0yMi4xMTQsNTguNTQxLTEuMjUxLDc4LjI2NGMtMTAuMzEzLTkuOTM1LTEwLjA5Mi0zMC42MDgsMC4zMzEtNTAuMzg3ICAgYzEuMzQ1LTEuNzc4LDUuMjMxLTUuOTYxLDEwLjkwNi00LjA1NWMwLjU2NSwwLjE5NSwwLjk0NiwwLjI0MiwxLjE2NywwLjIxMmMxLjI4MywwLjI3NSwyLjYwMywwLjQ0LDMuOTU5LDAuMzc3ICAgYzguNjIzLTAuMzkyLDE1LjI4OC03LjY4NCwxNC44OTktMTYuMzAzYy0wLjE3OS0zLjg2Ny0xLjc4Mi03LjI5Ni00LjI1Mi05LjkyYzE5Ljk5NS0xMy4xMDcsNDIuOTU5LTE0LjYxNyw1Mi4zNjYtNi4wMjEgICBsMC4zNTUsMC4wM0MxMzMuODkxLDE0NS4wODYsOTguNzUxLDE0OC41OTEsNzYuODk0LDE3Mi41MnoiIGZpbGw9ImJsYWNrIi8+DQogICAgICAgIDxwYXRoIGQ9Ik0zMDcuMzksMjkxLjE0NWMtMjEuNjY2LDIzLjcwNC0yMi4xMTgsNTguNTUzLTEuMjU0LDc4LjI3NmMtMTAuMzE3LTkuOTM1LTEwLjA5LTMwLjYxMSwwLjMzMS01MC4zODggICBjMS4zNDQtMS43ODEsNS4yMjctNS45NiwxMC45MDYtNC4wNTNjMC41NjgsMC4xODksMC45NDEsMC4yNDIsMS4xNjYsMC4yMTJjMS4yODIsMC4yNzcsMi42MDUsMC40NCwzLjk2MywwLjM4MSAgIGM4LjYxNy0wLjM5NiwxNS4yOC03LjY5MiwxNC44OTEtMTYuMzJjLTAuMTc2LTMuODY0LTEuNzgxLTcuMjg4LTQuMjUyLTkuOTE3YzIwLjAwMi0xMy4xMDMsNDIuOTU5LTE0LjYxLDUyLjM3LTYuMDE2bDAuMzU1LDAuMDI5ICAgQzM2NC4zODMsMjYzLjcxOCwzMjkuMjQxLDI2Ny4yMjYsMzA3LjM5LDI5MS4xNDV6IiBmaWxsPSJibGFjayIvPg0KICAgIDwvbWFzaz4NCiAgICA8cmVjdCB5PSIwIiB3aWR0aD0iNTI3LjMzNSIgaGVpZ2h0PSI1MjcuMzM1IiBmaWxsPSJjdXJyZW50Q29sb3IiIG1hc2s9InVybCgjbSkiLz4NCiAgICA8cmVjdCB5PSIzMTAiIHg9IjI1IiBmaWxsPSJjdXJyZW50Q29sb3IiIHdpZHRoPSI1NiIgaGVpZ2h0PSI1NiIgLz4NCiAgICA8Y2lyY2xlIHI9IjI4IiBjeD0iMTE5IiBjeT0iMzM4IiBmaWxsPSJjdXJyZW50Q29sb3IiIC8+DQogICAgPHBvbHlnb24gcG9pbnRzPSIxODIsMzEwIDE1MCwzNjYgMjE2LDM2NiIgZmlsbD0iY3VycmVudENvbG9yIiAvPg0KPC9zdmc+DQo=", "properties": { "data_graph_iri": { "name": "data_graph_iri", "title": "Input data graph", - "description": "The knowledge graph containing the instance data to be analyzed for the SHACL shapes generation.", + "description": "The knowledge graph holding the instance data to analyze.", "type": "string", "parameterType": "scheme:string", "value": null, @@ -3038,7 +3332,7 @@ "shapes_graph_iri": { "name": "shapes_graph_iri", "title": "Output shape catalog", - "description": "The knowledge graph the generated shapes will be added to.", + "description": "The graph the generated shapes are written to.", "type": "string", "parameterType": "scheme:string", "value": null, @@ -3046,21 +3340,10 @@ "visibleInDialog": true, "properties": {} }, - "label": { - "name": "label", - "title": "Output shape catalog label", - "description": "The label for the shape catalog graph. If no label is specified for a new shapes graph, a label will be generated. If no label is specified when adding to a shapes graph, the original label will be kept, or, if the existing graph does not have a label, a label will be generated. Only labels with language tag \"en\" or without language tag are considered.", - "type": "string", - "parameterType": "string", - "value": "", - "advanced": false, - "visibleInDialog": true, - "properties": {} - }, "existing_graph": { "name": "existing_graph", - "title": "Handle existing output graph", - "description": "Add result to the existing graph (add result to graph), overwrite the existing graph with the result (replace existing graph with result), or stop the workflow if the output graph already exists (stop workflow if output graph exists).", + "title": "Handle an existing shape catalog", + "description": "What to do when the output shape catalog already exists.", "type": "string", "parameterType": "string", "value": "stop", @@ -3070,8 +3353,8 @@ }, "import_shapes": { "name": "import_shapes", - "title": "Import the output graph into the central shapes catalog", - "description": "Import the SHACL shapes graph in the CMEM shapes catalog by adding an `owl:imports` statement to the central CMEM shapes catalog. If the graph is not imported, the new shapes are not activated and used.", + "title": "Import into the central shape catalog", + "description": "If enabled, the generated catalog is imported into the central shape catalog by adding an `owl:imports` statement to it. Shapes in a catalog the central one does not import are not picked up.", "type": "string", "parameterType": "boolean", "value": "false", @@ -3081,10 +3364,21 @@ } }, "properties_advanced": { + "label": { + "name": "label", + "title": "Output shape catalog label", + "description": "The label of the shape catalog. Left empty, a new catalog gets a generated label, and a catalog being added to keeps the label it has - but a catalog being replaced is rewritten whole, generated label included. Only a label tagged \"en\" or carrying no language tag counts as an existing label, and only such a label is replaced.", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": true, + "visibleInDialog": true, + "properties": {} + }, "prefix_cc": { "name": "prefix_cc", "title": "Fetch namespace prefixes from prefix.cc", - "description": "Fetch the list of namespace prefixes from https://prefix.cc instead of using the local prefix database. If unavailable, fall back to the local database. Prefixes defined in the Corporate Memory project override database prefixes. Enabling this option exposes your IP address to prefix.cc but no other data is shared. If unsure, keep this option disabled. See https://prefix.cc/about.", + "description": "If enabled, the namespace prefix list is fetched from https://prefix.cc instead of the local prefix database, falling back to the local database when the service cannot be reached. Prefixes defined in the project win over both. Enabling this exposes your IP address to prefix.cc; no other data is shared. If unsure, leave it disabled. See https://prefix.cc/about.", "type": "string", "parameterType": "boolean", "value": "false", @@ -3095,7 +3389,7 @@ "ignore_properties": { "name": "ignore_properties", "title": "Properties to ignore", - "description": "Provide the list of properties (as IRIs) to ignore.", + "description": "The properties to leave out of the generated shapes, as IRIs, one per line.", "type": "string", "parameterType": "multiline string", "value": "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", @@ -3105,8 +3399,8 @@ }, "ignore_types": { "name": "ignore_types", - "title": "Types to ignore", - "description": "Provide the list of types (as IRIs) to ignore.", + "title": "Classes to ignore", + "description": "The classes to leave out of the generated shapes, as IRIs, one per line.", "type": "string", "parameterType": "multiline string", "value": "", @@ -3116,33 +3410,146 @@ }, "plugin_provenance": { "name": "plugin_provenance", - "title": "Include plugin provenance", - "description": "Add information about the plugin and plugin settings to the shapes graph.", + "title": "Include task provenance", + "description": "If enabled, the shape catalog also records the task that generated it and the parameter values it ran with. Where the type of the task cannot be determined, nothing is recorded and the run continues with a warning in the log.", "type": "string", "parameterType": "boolean", "value": "false", "advanced": true, "visibleInDialog": true, "properties": {} - } - }, - "actions": {}, - "required": [ - "data_graph_iri", - "shapes_graph_iri" - ], - "distanceMeasureRange": null, - "backendType": "python", - "is_deprecated": false, - "tags": [ - "WorkflowTask", - "PythonPlugin" - ], - "pluginType": "customtask", - "relatedPlugins": [] - }, - "cmem-plugin-email-GetEmails": { - "pluginId": "cmem-plugin-email-GetEmails", + }, + "omit_namespace_addon": { + "name": "omit_namespace_addon", + "title": "Omit the namespace prefix from property names", + "description": "If enabled, property shape names and labels leave off the trailing namespace prefix, reading \"label\" rather than \"label (rdfs:)\". Node shape names always keep it.", + "type": "string", + "parameterType": "boolean", + "value": "false", + "advanced": true, + "visibleInDialog": true, + "properties": {} + }, + "managed_classes": { + "name": "managed_classes", + "title": "Declare the classes the catalog manages", + "description": "If enabled, the shape catalog states with `shui:managedClasses` which classes it manages: node shapes, property shapes, property groups, SPARQL constraints, prefix declarations, chart visualizations, table reports, widget integrations and workflow triggers.", + "type": "string", + "parameterType": "boolean", + "value": "true", + "advanced": true, + "visibleInDialog": true, + "properties": {} + }, + "depictions": { + "name": "depictions", + "title": "Add depictions to node shapes", + "description": "If enabled, a node shape is given the `foaf:depiction` of its target class, where one can be found. The class is looked up in the data graph and in the graph named by its own namespace, tried both with and without the trailing separator, because the graph a vocabulary was loaded into is not recorded anywhere. A class depicted more than once contributes one of its depictions, the same one on every run.", + "type": "string", + "parameterType": "boolean", + "value": "true", + "advanced": true, + "visibleInDialog": true, + "properties": {} + } + }, + "actions": { + "get_classes": { + "label": "Get classes", + "description": "Lists the classes used in the input data graph, one IRI per line, ready to paste into Classes to ignore.", + "icon": null + }, + "get_properties": { + "label": "Get properties", + "description": "Lists the properties used in the input data graph, one IRI per line, ready to paste into Properties to ignore.", + "icon": null + } + }, + "required": [ + "data_graph_iri", + "shapes_graph_iri" + ], + "distanceMeasureRange": null, + "backendType": "python", + "is_deprecated": false, + "tags": [ + "WorkflowTask", + "PythonPlugin" + ], + "pluginType": "customtask", + "relatedPlugins": [] + }, + "cmem_plugin_querycatalog-GetCatalogQuery": { + "pluginId": "cmem_plugin_querycatalog-GetCatalogQuery", + "title": "Get Catalog Query", + "categories": [ + "Uncategorized" + ], + "main_category": "Uncategorized", + "description": "Emit one entity describing a query stored in a query catalog.", + "markdownDocumentation": "\nThis task looks a query up in a query catalog and emits one entity describing it,\nwithout running it.\n\nThe entity carries the catalog graph, the query IRI, its label and description, its\ntype, its query text as stored, and the names of the placeholders that text contains.\nThe query text is emitted unfilled, with its placeholders intact, which is what makes\nit useful to a task that wants to inspect or transform a query rather than execute one.\n\nWith a query chosen on the task, it emits exactly one entity and takes no input. With\nthe query left empty, the queries to look up arrive on an input port instead, as a\ncatalog graph and a query IRI per entity, and one entity is emitted for each. An\nincoming entity that leaves the catalog graph empty falls back to the catalog chosen\non the task, so a list of query IRIs from one catalog needs no constant column.\n\nIt emits exactly what **List Catalog Queries** emits, in the same order, and the\nfirst two values it emits are what it accepts on its own input port - so the two tasks\nchain directly. **Execute Catalog Query** is the one that runs a query.\n\nA query IRI that the named catalog does not hold fails the task rather than being\nskipped, because a workflow asking for a specific query and silently receiving nothing\nis the harder failure to notice.\n", + "pluginIcon": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAzMiAzMiIgZmlsbD0iY3VycmVudENvbG9yIj4KICA8cGF0aCBkPSJNMjAuOTM3NSwxOS43NTI1aC0xLjE4NXYxLjk3NUgxOC4zN0wyMC4zNDUsMjQuMjk1bDEuOTc1LC0yLjU2NzVIMjAuOTM3NVoiLz4KICA8cGF0aCBkPSJNMjYuNDg4OCwyMS41MTNBNi44MjU2LDYuODI1NiwwLDAsMCwyMC4zNDUsMTcuMTg1YTYuODI1Niw2LjgyNTYsMCwwLDAsLTYuMTQzOCw0LjMyOEwxNC4wMjUsMjEuOTI1bDAuMTc2MiwwLjQxMkE2LjgyNTYsNi44MjU2LDAsMCwwLDIwLjM0NSwyNi42NjVhNi44MjU2LDYuODI1NiwwLDAsMCw2LjE0MzgsLTQuMzI4TDI2LjY2NSwyMS45MjVaTTIwLjM0NSwyNS4wODVhMy4xNiwzLjE2LDAsMSwxLDMuMTYsLTMuMTZBMy4xNjM2LDMuMTYzNiwwLDAsMSwyMC4zNDUsMjUuMDg1WiIvPgogIDxjaXJjbGUgY3g9IjkuMjg1IiBjeT0iOS4yODUiIHI9IjAuNzkiLz4KICA8Y2lyY2xlIGN4PSI5LjI4NSIgY3k9IjE1LjYwNSIgcj0iMC43OSIvPgogIDxjaXJjbGUgY3g9IjkuMjg1IiBjeT0iMjEuOTI1IiByPSIwLjc5Ii8+CiAgPHBhdGggZD0iTTYuOTE1LDE5LjU1NWg1LjUzVjE3Ljk3NUg2LjkxNVYxMy4yMzVIMTkuNTU1djIuMzdoMS41OFY2LjkxNWExLjU4LDEuNTgsMCwwLDAsLTEuNTgsLTEuNThINi45MTVBMS41OCwxLjU4LDAsMCwwLDUuMzM1LDYuOTE1VjI0LjI5NWExLjU4LDEuNTgsMCwwLDAsMS41OCwxLjU4aDUuNTNWMjQuMjk1SDYuOTE1Wk02LjkxNSw2LjkxNUgxOS41NTV2NC43NEg2LjkxNVoiLz4KPC9zdmc+Cg==", + "properties": { + "catalog_graph": { + "name": "catalog_graph", + "title": "Catalog Graph", + "description": "The graph the query is read from. The list offers the query catalogs first, then the shape catalogs, then any other graph that holds queries, with the number of read and write queries each one contains.", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "query_kind": { + "name": "query_kind", + "title": "Query Kind", + "description": "Which half of the catalog the query list offers.", + "type": "string", + "parameterType": "string", + "value": "all", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "query": { + "name": "query", + "title": "Query", + "description": "The query to use, picked by its label. Choose the catalog graph and the query kind first; changing either of them clears this field.", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + } + }, + "properties_advanced": {}, + "actions": { + "preview": { + "label": "Preview", + "description": "Show the query this task would emit.", + "icon": null + }, + "open_in_editor": { + "label": "Open in query editor", + "description": "Link the chosen query in the query editor.", + "icon": null + } + }, + "required": [], + "distanceMeasureRange": null, + "backendType": "python", + "is_deprecated": false, + "tags": [ + "WorkflowTask", + "PythonPlugin" + ], + "pluginType": "customtask", + "relatedPlugins": [] + }, + "cmem-plugin-email-GetEmails": { + "pluginId": "cmem-plugin-email-GetEmails", "title": "Get emails", "categories": [ "Uncategorized" @@ -3260,6 +3667,288 @@ "pluginType": "customtask", "relatedPlugins": [] }, + "cmem_plugin_ntfy-GetAttachments": { + "pluginId": "cmem_plugin_ntfy-GetAttachments", + "title": "Get ntfy attachments", + "categories": [ + "Uncategorized" + ], + "main_category": "Uncategorized", + "description": "Download the attachments of a topic's messages from an ntfy server.", + "markdownDocumentation": "## What it does\n\nDownloads the files attached to the messages of a topic, so a workflow can\nprocess what somebody sent to it.\n\n## What flows through the ports\n\nThe task consumes nothing and reads from the server itself. Its output port\ncarries one file entity per downloaded attachment, with the media type the\nmessage declared and the name it gave, reduced to a plain file name. A file\ndataset or a file consuming task can be connected to it.\n\n## Where it sits\n\nIt reads the same topic as **Get ntfy messages** and keeps its own position, so\nthe two can run side by side without taking each other's messages. They share\none position file by default, which is worth separating if they are scheduled\nto finish at the same moment. Use that task instead when the text of the\nmessages is what matters.\n\n## Caveats\n\n- **A message without an attachment is passed over**, and so is an attachment\n that is not where the message said - ntfy drops an uploaded file after three\n hours. Neither fails the run, and the report counts them apart, because the\n second one is also what a wrong address looks like from here.\n- **A run either reads everything or only what is new**, as with the messages\n task, and the position advances over passed-over messages as well, so they\n are not offered again.\n- **A message published a moment ago may not be there yet.** A busy server\n writes its message cache in batches - ntfy.sh is configured for one second -\n so a workflow that publishes to a topic and reads it back in the same run can\n come away with nothing. Reading what another run published is reliable;\n reading what this run published is not.\n- **An attachment is fetched from wherever the message points.** A publisher\n may attach a file that is hosted elsewhere, and ntfy hands such a URL on\n unchanged, so the download goes to that host rather than to the configured\n server. The access token is sent to the configured server only.\n- **ntfy writes those addresses from its own base URL**, not from the address\n it was reached on, so a server behind a container host name, a proxy or a\n tunnel hands out links that point somewhere this task cannot reach - and that\n it does not recognise as the configured server either, which withholds the\n token as well. The task can be told to fetch ntfy's own file path from the\n configured server instead, which leaves an attachment hosted elsewhere alone.\n- **Files are downloaded to the worker's temporary directory** and are not\n cleaned up by the task, because the rest of the workflow still has to read\n them.\n- **An attachment may be large, and only this task limits it.** ntfy allows\n 15 MB per file by default, but an attachment hosted elsewhere is not bound by\n that, so anything over the size limit configured here is passed over and\n counted rather than written to the worker's disk.\n- **An unprotected topic is public, and anybody may write to it.** A topic\n needs no sign-up and carries no password of its own, so on a server without\n access control anybody who guesses the name can publish to it - which,\n together with the point above, means choosing what this task downloads and\n from where. Give the topic a long random suffix rather than a word like\n `reports`, and protect it with the server's own\n [access control](https://docs.ntfy.sh/config/#access-control) where that is available.\n", + "pluginIcon": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIyNCIgaGVpZ2h0PSIyNCIgdmlld0JveD0iMCAwIDI0IDI0IiBmaWxsPSJjdXJyZW50Q29sb3IiIGZpbGwtcnVsZT0iZXZlbm9kZCIgZm9jdXNhYmxlPSJmYWxzZSIgYXJpYS1oaWRkZW49InRydWUiPgo8ZyB0cmFuc2Zvcm09InRyYW5zbGF0ZSgtMS42IDEuMSkgc2NhbGUoMC43OCkiPjxwYXRoIGQ9Ik02IDIuMmg3LjNMMTkuNiA4LjV2MTMuM0g2eiIvPjxwYXRoIGQ9Ik0xNC4yIDIuNHY1LjJoNS4yeiIvPjwvZz48cGF0aCBkPSJNMTkuNiAxOC40IDE2IDEzLjloMi4zVjcuNmgyLjZ2Ni4zSDIzeiIvPgo8L3N2Zz4K", + "properties": { + "server": { + "name": "server", + "title": "Server", + "description": "The base URL of the ntfy server. The default is the public service; a self-hosted ntfy is named here instead, for example https://ntfy.example.org - see [installing ntfy](https://docs.ntfy.sh/install/).", + "type": "string", + "parameterType": "string", + "value": "https://ntfy.sh", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "access_token": { + "name": "access_token", + "title": "Access token", + "description": "The access token to authenticate with. A public topic needs none, so this can be left empty. See [access tokens](https://docs.ntfy.sh/config/#access-tokens).", + "type": "string", + "parameterType": "password", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "topic": { + "name": "topic", + "title": "Topic", + "description": "The topic whose attachments are downloaded. A topic needs no sign-up and has no password of its own, so give it a long random suffix nobody can guess rather than a plain word - see [picking a topic](https://docs.ntfy.sh/publish/#picking-a-topic).", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "read_mode": { + "name": "read_mode", + "title": "Messages to return", + "description": "Which of a topic's messages a run returns.", + "type": "string", + "parameterType": "string", + "value": "stateless", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "since": { + "name": "since", + "title": "Starting point", + "description": "How far back a run reaches: a duration such as `10m`, a Unix timestamp, a message id, or `all` for everything the server still holds. A task set to return only new messages reads this on its first run only, and continues from its remembered position afterwards. See [fetching cached messages](https://docs.ntfy.sh/subscribe/api/#fetch-cached-messages).", + "type": "string", + "parameterType": "string", + "value": "all", + "advanced": false, + "visibleInDialog": true, + "properties": {} + } + }, + "properties_advanced": { + "priority_filter": { + "name": "priority_filter", + "title": "Priority filter", + "description": "Return only messages of these priorities, as a comma separated list - each one a number from 1 to 5 or one of ntfy's names, from `min` to `max`. A message matching any of them is returned. Empty returns every priority. See [filtering messages](https://docs.ntfy.sh/subscribe/api/#filter-messages).", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": true, + "visibleInDialog": true, + "properties": {} + }, + "tags_filter": { + "name": "tags_filter", + "title": "Tag filter", + "description": "Return only messages carrying these tags, as a comma separated list. A message must carry all of them. Empty returns every message. See [filtering messages](https://docs.ntfy.sh/subscribe/api/#filter-messages).", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": true, + "visibleInDialog": true, + "properties": {} + }, + "maximum_size": { + "name": "maximum_size", + "title": "Maximum attachment size", + "description": "How many megabytes an attachment may weigh before it is passed over rather than downloaded. An attachment URL is whatever the publisher of the message put there, and ntfy forwards an external one unchanged, so the server's own limit of 15 MB does not bind it. 0 downloads whatever arrives.", + "type": "string", + "parameterType": "Long", + "value": "100", + "advanced": true, + "visibleInDialog": true, + "properties": {} + }, + "from_own_server": { + "name": "from_own_server", + "title": "Fetch attachments from this server", + "description": "If enabled, an attachment is fetched from the server configured above rather than from the address the message carries. ntfy builds that address from its own base URL, so a server reached under another name - a container host name, a proxy, a tunnel - hands out links that point somewhere unreachable and that the task does not recognise as its own, which also withholds the access token. Only ntfy's own file path is redirected; an attachment the publisher hosted elsewhere is always fetched from where it says.", + "type": "string", + "parameterType": "boolean", + "value": "false", + "advanced": true, + "visibleInDialog": true, + "properties": {} + }, + "cursor_file": { + "name": "cursor_file", + "title": "Position file", + "description": "The project file the read positions are kept in. One file holds the positions of every task in the project and is written whole, so give a task its own file when it may finish at the same moment as another one reading the same project.", + "type": "string", + "parameterType": "string", + "value": "ntfy-cursors.json", + "advanced": true, + "visibleInDialog": true, + "properties": {} + } + }, + "actions": { + "validate_connection": { + "label": "Validate connection", + "description": "Ask the server who the configured token belongs to.", + "icon": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIyNCIgaGVpZ2h0PSIyNCIgdmlld0JveD0iMCAwIDI0IDI0IiBmaWxsPSJjdXJyZW50Q29sb3IiIGZpbGwtcnVsZT0iZXZlbm9kZCIgZm9jdXNhYmxlPSJmYWxzZSIgYXJpYS1oaWRkZW49InRydWUiPgo8cGF0aCBkPSJNOS40IDE4LjIgMy4xIDExLjlsMi4zLTIuMyA0IDRMMTguNiA0bDIuMyAyLjN6Ii8+Cjwvc3ZnPgo=" + }, + "preview_messages": { + "label": "Preview messages", + "description": "Show the first messages the starting point names.", + "icon": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIyNCIgaGVpZ2h0PSIyNCIgdmlld0JveD0iMCAwIDI0IDI0IiBmaWxsPSJjdXJyZW50Q29sb3IiIGZpbGwtcnVsZT0iZXZlbm9kZCIgZm9jdXNhYmxlPSJmYWxzZSIgYXJpYS1oaWRkZW49InRydWUiPgo8cGF0aCBkPSJNMTIgNUM2LjggNSAyLjQgOC4xLjYgMTIuNWMxLjggNC40IDYuMiA3LjUgMTEuNCA3LjVzOS42LTMuMSAxMS40LTcuNUMyMS42IDguMSAxNy4yIDUgMTIgNXptMCAxMi4zYTQuOCA0LjggMCAxIDEgMC05LjYgNC44IDQuOCAwIDAgMSAwIDkuNnoiLz48Y2lyY2xlIGN4PSIxMiIgY3k9IjEyLjUiIHI9IjIuNCIvPgo8L3N2Zz4K" + }, + "reset_cursor": { + "label": "Forget the position", + "description": "Make the next run start from the starting point again.", + "icon": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIyNCIgaGVpZ2h0PSIyNCIgdmlld0JveD0iMCAwIDI0IDI0IiBmaWxsPSJjdXJyZW50Q29sb3IiIGZpbGwtcnVsZT0iZXZlbm9kZCIgZm9jdXNhYmxlPSJmYWxzZSIgYXJpYS1oaWRkZW49InRydWUiPgo8cGF0aCBkPSJNMTcuNiA2LjNBOCA4IDAgMSAwIDE5LjcgMTRoLTIuMUE2IDYgMCAxIDEgMTIgNmMxLjcgMCAzLjIuNyA0LjIgMS44TDEzIDExaDdWNHoiLz4KPC9zdmc+Cg==" + } + }, + "required": [], + "distanceMeasureRange": null, + "backendType": "python", + "is_deprecated": false, + "tags": [ + "WorkflowTask", + "PythonPlugin" + ], + "pluginType": "customtask", + "relatedPlugins": [] + }, + "cmem_plugin_ntfy-GetMessages": { + "pluginId": "cmem_plugin_ntfy-GetMessages", + "title": "Get ntfy messages", + "categories": [ + "Uncategorized" + ], + "main_category": "Uncategorized", + "description": "Read the messages of a topic from an ntfy server.", + "markdownDocumentation": "## What it does\n\nReads the messages a topic holds and hands them on as entities, so a workflow\ncan act on what arrived through ntfy.\n\n## What flows through the ports\n\nThe task consumes nothing and reads from the server itself. Its output port\ncarries one entity per message, with the paths `id`, `time`, `time_iso`,\n`topic`, `title`, `message`, `priority`, `tags`, `click`, `icon`, the four\n`attachment_*` paths, and `json` holding the message verbatim - so a field this\ntask does not model can still be read downstream. `tags` is the only path that\nmay carry several values; every other path carries one or none. A message that\nwas sent without a priority carries 3, the default ntfy applies to it and\nleaves out of its answer.\n\n## Where it sits\n\nIt is the reading half of this package. **Get ntfy attachments** fetches the\nfiles of those messages, and **Send ntfy message** sends in the other\ndirection.\n\n## Caveats\n\n- **A run either returns everything or only what is new.** Returning only new\n ones makes the task write the position it reached into a file in the project\n and continue from it next time. Each task has its own entry there, so two\n tasks reading one topic never take each other's messages - but the file is\n written whole, so two of them finishing in the same moment can lose one of\n the two positions, and that task reads its window again.\n- **The position only advances over messages a consumer asked past.** Taking\n an entity is not proof that it was processed, so the position follows the\n last message the rest of the workflow came back from. A run failing halfway\n leaves everything after that to be read again: a message is delivered at\n least once, duplicates are possible, and a lost message is not. A worker\n killed outright writes no position at all.\n- **A message published a moment ago may not be there yet.** A busy server\n writes its message cache in batches - ntfy.sh is configured for one second -\n so a workflow that publishes to a topic and reads it back in the same run can\n come away with nothing. Reading what another run published is reliable;\n reading what this run published is not.\n- **A message id that the server has forgotten cannot be resumed from.** ntfy\n keeps messages for twelve hours by default, so a task that has not run for\n longer starts from the whole window again - duplicates rather than loss.\n- **The preview shows what the starting point names**, not the position a run\n would continue from, because a task action is not told which task it belongs\n to. Resetting the position has the same limitation and clears the position of\n every task reading this topic in this project.\n- **An unprotected topic is public, and anybody may write to it.** A topic\n needs no sign-up and carries no password of its own, so on a server without\n access control anybody who guesses the name can read what is on it and\n publish to it - and this task hands a stranger's message to the workflow as\n readily as its own. Give the topic a long random suffix rather than a word\n like `alerts`, and protect it with the server's own\n [access control](https://docs.ntfy.sh/config/#access-control) where that is available.\n", + "pluginIcon": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIyNCIgaGVpZ2h0PSIyNCIgdmlld0JveD0iMCAwIDI0IDI0IiBmaWxsPSJjdXJyZW50Q29sb3IiIGZpbGwtcnVsZT0iZXZlbm9kZCIgZm9jdXNhYmxlPSJmYWxzZSIgYXJpYS1oaWRkZW49InRydWUiPgo8ZyB0cmFuc2Zvcm09InRyYW5zbGF0ZSgtMS42IDEuMSkgc2NhbGUoMC43OCkiPjxwYXRoIGQ9Ik0xMiAyMi4yYTIuMyAyLjMgMCAwIDAgMi4zLTIuMkg5LjdhMi4zIDIuMyAwIDAgMCAyLjMgMi4yeiIvPjxwYXRoIGQ9Ik0xOSAxNi40di01LjNjMC0zLjMtMS44LTYtNC44LTYuN3YtLjdhMi4yIDIuMiAwIDAgMC00LjQgMHYuN0M2LjggNS4xIDUgNy44IDUgMTEuMXY1LjNMMi44IDE4LjZ2MS4xaDE4LjR2LTEuMXoiLz48L2c+PHBhdGggZD0iTTE5LjYgMTguNCAxNiAxMy45aDIuM1Y3LjZoMi42djYuM0gyM3oiLz4KPC9zdmc+Cg==", + "properties": { + "server": { + "name": "server", + "title": "Server", + "description": "The base URL of the ntfy server. The default is the public service; a self-hosted ntfy is named here instead, for example https://ntfy.example.org - see [installing ntfy](https://docs.ntfy.sh/install/).", + "type": "string", + "parameterType": "string", + "value": "https://ntfy.sh", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "access_token": { + "name": "access_token", + "title": "Access token", + "description": "The access token to authenticate with. A public topic needs none, so this can be left empty. See [access tokens](https://docs.ntfy.sh/config/#access-tokens).", + "type": "string", + "parameterType": "password", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "topic": { + "name": "topic", + "title": "Topic", + "description": "The topic whose messages are read. A topic needs no sign-up and has no password of its own, so give it a long random suffix nobody can guess rather than a plain word - see [picking a topic](https://docs.ntfy.sh/publish/#picking-a-topic).", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "read_mode": { + "name": "read_mode", + "title": "Messages to return", + "description": "Which of a topic's messages a run returns.", + "type": "string", + "parameterType": "string", + "value": "stateless", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "since": { + "name": "since", + "title": "Starting point", + "description": "How far back a run reaches: a duration such as `10m`, a Unix timestamp, a message id, or `all` for everything the server still holds. A task set to return only new messages reads this on its first run only, and continues from its remembered position afterwards. See [fetching cached messages](https://docs.ntfy.sh/subscribe/api/#fetch-cached-messages).", + "type": "string", + "parameterType": "string", + "value": "all", + "advanced": false, + "visibleInDialog": true, + "properties": {} + } + }, + "properties_advanced": { + "priority_filter": { + "name": "priority_filter", + "title": "Priority filter", + "description": "Return only messages of these priorities, as a comma separated list - each one a number from 1 to 5 or one of ntfy's names, from `min` to `max`. A message matching any of them is returned. Empty returns every priority. See [filtering messages](https://docs.ntfy.sh/subscribe/api/#filter-messages).", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": true, + "visibleInDialog": true, + "properties": {} + }, + "tags_filter": { + "name": "tags_filter", + "title": "Tag filter", + "description": "Return only messages carrying these tags, as a comma separated list. A message must carry all of them. Empty returns every message. See [filtering messages](https://docs.ntfy.sh/subscribe/api/#filter-messages).", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": true, + "visibleInDialog": true, + "properties": {} + }, + "cursor_file": { + "name": "cursor_file", + "title": "Position file", + "description": "The project file the read positions are kept in. One file holds the positions of every task in the project and is written whole, so give a task its own file when it may finish at the same moment as another one reading the same project.", + "type": "string", + "parameterType": "string", + "value": "ntfy-cursors.json", + "advanced": true, + "visibleInDialog": true, + "properties": {} + } + }, + "actions": { + "validate_connection": { + "label": "Validate connection", + "description": "Ask the server who the configured token belongs to.", + "icon": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIyNCIgaGVpZ2h0PSIyNCIgdmlld0JveD0iMCAwIDI0IDI0IiBmaWxsPSJjdXJyZW50Q29sb3IiIGZpbGwtcnVsZT0iZXZlbm9kZCIgZm9jdXNhYmxlPSJmYWxzZSIgYXJpYS1oaWRkZW49InRydWUiPgo8cGF0aCBkPSJNOS40IDE4LjIgMy4xIDExLjlsMi4zLTIuMyA0IDRMMTguNiA0bDIuMyAyLjN6Ii8+Cjwvc3ZnPgo=" + }, + "preview_messages": { + "label": "Preview messages", + "description": "Show the first messages the starting point names.", + "icon": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIyNCIgaGVpZ2h0PSIyNCIgdmlld0JveD0iMCAwIDI0IDI0IiBmaWxsPSJjdXJyZW50Q29sb3IiIGZpbGwtcnVsZT0iZXZlbm9kZCIgZm9jdXNhYmxlPSJmYWxzZSIgYXJpYS1oaWRkZW49InRydWUiPgo8cGF0aCBkPSJNMTIgNUM2LjggNSAyLjQgOC4xLjYgMTIuNWMxLjggNC40IDYuMiA3LjUgMTEuNCA3LjVzOS42LTMuMSAxMS40LTcuNUMyMS42IDguMSAxNy4yIDUgMTIgNXptMCAxMi4zYTQuOCA0LjggMCAxIDEgMC05LjYgNC44IDQuOCAwIDAgMSAwIDkuNnoiLz48Y2lyY2xlIGN4PSIxMiIgY3k9IjEyLjUiIHI9IjIuNCIvPgo8L3N2Zz4K" + }, + "reset_cursor": { + "label": "Forget the position", + "description": "Make the next run start from the starting point again.", + "icon": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIyNCIgaGVpZ2h0PSIyNCIgdmlld0JveD0iMCAwIDI0IDI0IiBmaWxsPSJjdXJyZW50Q29sb3IiIGZpbGwtcnVsZT0iZXZlbm9kZCIgZm9jdXNhYmxlPSJmYWxzZSIgYXJpYS1oaWRkZW49InRydWUiPgo8cGF0aCBkPSJNMTcuNiA2LjNBOCA4IDAgMSAwIDE5LjcgMTRoLTIuMUE2IDYgMCAxIDEgMTIgNmMxLjcgMCAzLjIuNyA0LjIgMS44TDEzIDExaDdWNHoiLz4KPC9zdmc+Cg==" + } + }, + "required": [], + "distanceMeasureRange": null, + "backendType": "python", + "is_deprecated": false, + "tags": [ + "WorkflowTask", + "PythonPlugin" + ], + "pluginType": "customtask", + "relatedPlugins": [] + }, "getProjectFiles": { "pluginId": "getProjectFiles", "title": "Get project files", @@ -3331,7 +4020,7 @@ ], "main_category": "Uncategorized", "description": "Output a workflow execution report as a JSON file.", - "markdownDocumentation": "\nThis workflow operator retrieves a specific execution report of a workflow and outputs it\nas a JSON file.\n\nThe plugin queries the backend API to list all available reports for the given workflow and\nallows you to select which report to retrieve:\n\n- **Latest Report**: The most recent execution report\n- **Latest Report with Errors**: The most recent failed execution report\n- **Latest Report with Warning or Errors**: The most recent report with warnings (successful or not)\n\nThe report contains detailed information about the workflow execution, including task results,\nexecution times, and any errors or warnings.\n\n## Output\n\nThe plugin outputs a single JSON file entity containing the complete workflow execution report.\n\n## Usage\n\nThis operator is useful for:\n- Monitoring workflow execution results and failures\n- Debugging recent errors or warnings\n- Creating audit trails of workflow runs\n- Archiving execution reports for compliance purposes\n- Feeding execution data into downstream analysis tasks\n", + "markdownDocumentation": "\nThis workflow operator retrieves one execution report of a workflow and outputs it as a JSON\nfile.\n\n## Ports\n\nThe task has no input port; it queries the reports available for the selected workflow\ndirectly. It outputs a single JSON file entity holding the report - the same JSON payload\nreturned for that execution, including its finish status and any warnings.\n\n## Usage\n\nWith no input port, this operator is typically the first task in a workflow. Its output feeds\na downstream task that reads or forwards the JSON file, for example to inspect it, archive it,\nor route on the presence of errors or warnings.\n\n## Caveats\n\n- If no report matches the selected mode - none currently carries warnings or errors, or the\n time period excludes every report - the output file contains an empty JSON object `{}`, not\n an error or any other indication that nothing was found.\n- Finding the latest report with warnings or errors is checked from newest to oldest, and may\n fetch every report in scope individually before a match is found or the list is exhausted.\n", "pluginIcon": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAzMiAzMiI+PHBhdGggZD0iTTIyLDRWN0gxMFY0SDJ2OGg4VjloNy4wMjM0QTQuOTQ2Myw0Ljk0NjMsMCwwLDAsMTYsMTJ2OEEzLjAwMzcsMy4wMDM3LDAsMCwxLDEzLDIzSDEwVjIwSDJ2OGg4VjI1SDEzQTUuMDA2Miw1LjAwNjIsMCwwLDAsMTgsMjBWMTJhMi45OTgyLDIuOTk4MiwwLDAsMSwyLjk5NzEtM0gyMnYzaDhWNFpNOCwxMEg0VjZIOFpNOCwyNkg0VjIySDhaTTI4LDEwSDI0VjZoNFoiLz48L3N2Zz4=", "properties": { "workflow_id": { @@ -3359,19 +4048,174 @@ "time_period": { "name": "time_period", "title": "Time Period", - "description": "The time period in hours of the workflow execution reports that can be listed.Allows only full hours. Defaults to 0 for every execution report.", + "description": "The time period in hours of the workflow execution reports that can be listed. Allows only full hours. Defaults to 0, which lists every execution report.", + "type": "string", + "parameterType": "Long", + "value": "0", + "advanced": false, + "visibleInDialog": true, + "properties": {} + } + }, + "properties_advanced": {}, + "actions": {}, + "required": [ + "workflow_id" + ], + "distanceMeasureRange": null, + "backendType": "python", + "is_deprecated": false, + "tags": [ + "WorkflowTask", + "PythonPlugin" + ], + "pluginType": "customtask", + "relatedPlugins": [] + }, + "cmem_plugin_graphql-Query": { + "pluginId": "cmem_plugin_graphql-Query", + "title": "GraphQL query", + "categories": [ + "Uncategorized" + ], + "main_category": "Uncategorized", + "description": "Sends a GraphQL query or mutation to an endpoint and returns the result as entities.", + "markdownDocumentation": "This task sends a GraphQL query or mutation to an endpoint and\ncaptures the response.\n\n## How often the endpoint is called\n\nJinja syntax in the query or in the variables turns the task into a loop: both are\nrendered once per arriving entity and the endpoint is called once per entity, with a\nfailing entity logged, counted in the report and skipped rather than taking the whole\ntask down. Text without Jinja syntax is sent exactly once, and anything connected as\ninput is then ignored.\n\n## What leaves the task\n\nThe responses leave on the output port in one of two shapes:\n\n- **entities**: one per call. Their paths are the fields the query asks for, under the\n alias where a field has one, so the next task is offered the schema while the workflow\n is drawn rather than only after a first run. A field that selects sub fields becomes a\n relation, and the entities behind it follow the shape of the response.\n- **file**: all responses of the run written to a single JSON file. What leaves the port\n is that one file rather than the data in it.\n\nHow many values a field carries is not part of a query - that lives in the endpoint's own\nschema - so every path is offered as possibly multi valued and the entities are built to\nmatch. A field the endpoint answers with a single object therefore arrives as a list\nholding one entity, and a JSON dataset connected to the output port holds an array in\nthat place.\n\n## Where this task fits\n\nThe task usually opens a chain: a GraphQL API at one end, and at the other a transform\nthat maps the response into a graph, or a dataset that keeps it for later steps. The file\nshape suits the second kind of chain, and a task that uploads or stores what it is handed,\nsince a file travels through those without being taken apart on the way.\n\n## Caveats\n\n- **Three kinds of query describe nothing in advance**, and the entities then leave with a\n schema that stays unknown until the task has run, which the next task has to accept as\n it comes: one that does not parse as GraphQL on its own, which is the usual case for a\n Jinja template because the placeholders sit where GraphQL expects values; one holding\n more than one operation; and one whose top level is a fragment rather than plain fields.\n A run that hands on a file is not affected, since a file is described the same way\n whatever the query asks for.\n- **Jinja text is never checked for GraphQL syntax errors until it is rendered**, so a\n mistake in it surfaces while the task runs, as a failed entity, rather than as a\n configuration error while the task is set up.\n", + "pluginIcon": null, + "properties": { + "graphql_url": { + "name": "graphql_url", + "title": "Endpoint", + "description": "The URL the query is sent to, for example `https://fruits-api.netlify.app/graphql`. A collective list of public GraphQL APIs is available [here](https://github.com/IvanGoncharov/graphql-apis).", + "type": "string", + "parameterType": "string", + "value": null, + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "access_token": { + "name": "access_token", + "title": "Access token", + "description": "The token the endpoint is authenticated with. It is sent as the `Authorization: Bearer ` header. Leave it empty for an endpoint that needs no authentication. For GitLab, a personal, project or group access token works, with scope `read_api` for queries or `api` for mutations.", + "type": "string", + "parameterType": "password", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "output_mode": { + "name": "output_mode", + "title": "Output mode", + "description": "In which shape the responses leave this task.", + "type": "string", + "parameterType": "string", + "value": "entities", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "graphql_query": { + "name": "graphql_query", + "title": "Query", + "description": "The query or mutation to send, written in [GraphQL](https://graphql.org/). May contain Jinja syntax, for example `{{ id }}`, which is rendered against each arriving entity before the query is sent. `query allFruits { fruits { id scientific_name fruit_name family climatic_zone } }`", + "type": "string", + "parameterType": "multiline string", + "value": null, + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "graphql_variable_values": { + "name": "graphql_variable_values", + "title": "Query variables", + "description": "The values for the variables the query declares, as a JSON object - `{\"id\": 1}` for a query taking `$id`. An empty object is sent for a query that declares none. May contain Jinja syntax, for example `{\"id\": {{ id }}}`, which is rendered against each arriving entity before the query is sent.", + "type": "string", + "parameterType": "multiline string", + "value": "{}", + "advanced": false, + "visibleInDialog": true, + "properties": {} + } + }, + "properties_advanced": {}, + "actions": {}, + "required": [ + "graphql_url", + "graphql_query" + ], + "distanceMeasureRange": null, + "backendType": "python", + "is_deprecated": false, + "tags": [ + "WorkflowTask", + "PythonPlugin" + ], + "pluginType": "customtask", + "relatedPlugins": [] + }, + "cmem_plugin_hetzner-ListResources": { + "pluginId": "cmem_plugin_hetzner-ListResources", + "title": "Hetzner Cloud: List Resources", + "categories": [ + "Uncategorized" + ], + "main_category": "Uncategorized", + "description": "Inventory all existing resources of a Hetzner Cloud project (servers, volumes, networks, load balancers, IPs, firewalls, SSH keys, snapshots, \u2026), one entity per resource in a single union schema.", + "markdownDocumentation": "\nThis workflow operator inventories the resources actually provisioned in a\nHetzner Cloud project, using the [Hetzner Cloud API](https://docs.hetzner.cloud/).\nIt complements **Hetzner Cloud: List Server Types** (the catalog of what you\n*could* create) by listing what you *have*.\n\nIt produces one entity per resource across 11 resource types: `server`,\n`volume`, `network`, `load_balancer`, `floating_ip`, `primary_ip`, `firewall`,\n`ssh_key`, `image`, `certificate` and `placement_group`. All entities share a\nsingle union schema; the `resourceType` column discriminates each row's type,\nand each row fills its shared columns plus its own type's columns. A column\nthat does not apply to a row carries no value at all, rather than an empty one.\n\nImages are restricted to project-owned snapshots and backups; the shared\nsystem / app OS catalog is excluded. Since the Hetzner API names only system\nimages, the `name` of a snapshot or backup falls back to its description.\n\nNote that `ipv6` holds the routed IPv6 network in CIDR notation for a server,\nand a single address for the IP resources.\n\n## Authentication\n\nYou need a Hetzner Cloud API token of a project (Console \u2192 Security \u2192 API\nTokens). A read-only token is sufficient.\n\n## Filters\n\nAll filters are optional. Leave them empty to inventory everything. A filter\nthat matches nothing yields an empty result, which the execution report states.\n\nLocated resources are the server, volume, primary IP, load balancer and\nfloating IP (by its home location). The network, firewall, SSH key, image,\ncertificate and placement group are global in the Hetzner API and always pass\nthe location filter.\n", + "pluginIcon": "data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiIHN0YW5kYWxvbmU9Im5vIj8+CjxzdmcgZm9jdXNhYmxlPSJmYWxzZSIgcHJlc2VydmVBc3BlY3RSYXRpbz0ieE1pZFlNaWQgc2xpY2UiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgZmlsbD0iY3VycmVudENvbG9yIiB3aWR0aD0iMzIiIGhlaWdodD0iMzIiIHZpZXdCb3g9IjAgMCAzMiAzMiIgYXJpYS1oaWRkZW49InRydWUiIGNsYXNzPSJlY2NndWktaWNvbiI+CiAgPCEtLSBTaGFyZWQgYmFzZTogdGhlIEhldHpuZXIgSCwgaWRlbnRpY2FsIGluIGV2ZXJ5IGljb24gb2YgdGhpcyBwYWNrYWdlLgogICAgICAgQSB0YXNrIGljb24gaXMgdGhpcyBwYXRoIHBsdXMgb25lIG1vZGlmaWVyIGluIHRoZSBsb3dlciByaWdodCAxOC4uMzEKICAgICAgIHNxdWFyZS4gS2VlcCB0aGUgYmFzZSBieXRlIGZvciBieXRlIHdoZW4gYWRkaW5nIGFuIGljb24uIC0tPgogIDxwYXRoIGQ9Ik0yLDNoNHY3aDZWM2g0VjIxaC00VjE0SDZ2N0gyWiI+PC9wYXRoPgogIDwhLS0gTW9kaWZpZXI6IGZvdXIgYmxvY2tzLCBmb3IgdGhlIHJlc291cmNlcyBhIHByb2plY3QgaG9sZHMuIC0tPgogIDxwYXRoIGQ9Ik0xOCwxOGg2djZoLTZaIj48L3BhdGg+CiAgPHBhdGggZD0iTTI1LDE4aDZ2NmgtNloiPjwvcGF0aD4KICA8cGF0aCBkPSJNMTgsMjVoNnY2aC02WiI+PC9wYXRoPgogIDxwYXRoIGQ9Ik0yNSwyNWg2djZoLTZaIj48L3BhdGg+Cjwvc3ZnPgo=", + "properties": { + "api_token": { + "name": "api_token", + "title": "API Token", + "description": "A Hetzner Cloud project API token (read-only is sufficient).", + "type": "string", + "parameterType": "password", + "value": null, + "advanced": false, + "visibleInDialog": true, + "properties": {} + } + }, + "properties_advanced": { + "resource_type": { + "name": "resource_type", + "title": "Resource type", + "description": "Restrict to a single resource type. Empty lists all types.", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": true, + "visibleInDialog": true, + "properties": {} + }, + "label_selector": { + "name": "label_selector", + "title": "Label selector", + "description": "A Hetzner label selector (e.g. env=prod), applied server-side.", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": true, + "visibleInDialog": true, + "properties": {} + }, + "location": { + "name": "location", + "title": "Location", + "description": "Restrict located resources to a single location name (e.g. fsn1). Empty lists all. Global resources always pass through.", "type": "string", - "parameterType": "Long", - "value": "0", - "advanced": false, + "parameterType": "string", + "value": "", + "advanced": true, "visibleInDialog": true, "properties": {} } }, - "properties_advanced": {}, "actions": {}, "required": [ - "workflow_id" + "api_token" ], "distanceMeasureRange": null, "backendType": "python", @@ -3383,56 +4227,56 @@ "pluginType": "customtask", "relatedPlugins": [] }, - "cmem_plugin_graphql-workflow-graphql-GraphQLPlugin": { - "pluginId": "cmem_plugin_graphql-workflow-graphql-GraphQLPlugin", - "title": "GraphQL query", + "cmem_plugin_hetzner-ListServerTypes": { + "pluginId": "cmem_plugin_hetzner-ListServerTypes", + "title": "Hetzner Cloud: List Server Types", "categories": [ "Uncategorized" ], "main_category": "Uncategorized", - "description": "Executes a custom GraphQL query to a GraphQL endpoint and saves result to a JSON dataset.", - "markdownDocumentation": "This workflow task performs GraphQL operations by sending\n queries, mutations, and variables over operations. Allows for customization\n in the GraphQL query using, Jinja queries and Jinja variables, which can be\n obtained from entities. The result of the query is saved as a JSON document\n in a pre-created JSON dataset.\n ", - "pluginIcon": null, + "description": "List all available Hetzner Cloud server (VM) types, one entity per server type and location, including specs and per-location pricing.", + "markdownDocumentation": "\nThis workflow operator lists the available server (virtual machine) types of the\nHetzner Cloud, using the [Hetzner Cloud API](https://docs.hetzner.cloud/).\n\nIt produces one entity per **(server type x location)** offering. Each entity\ncarries the server type specification (cores, memory, disk, CPU type,\narchitecture, \u2026), the per-location availability, and the per-location price.\n\nBooleans are written in their XSD form (`true` / `false`), and a property a\nserver type does not carry is left out of the entity rather than written as an\nempty value.\n\n## Authentication\n\nYou need a Hetzner Cloud API token of a project (Console \u2192 Security \u2192 API\nTokens). A read-only token is sufficient.\n\n## Filters\n\nAll filters are optional. Leave them empty (or `false`) to list everything. A\nfilter that matches nothing yields an empty result, which the execution report\nstates.\n", + "pluginIcon": "data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiIHN0YW5kYWxvbmU9Im5vIj8+CjxzdmcgZm9jdXNhYmxlPSJmYWxzZSIgcHJlc2VydmVBc3BlY3RSYXRpbz0ieE1pZFlNaWQgc2xpY2UiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgZmlsbD0iY3VycmVudENvbG9yIiB3aWR0aD0iMzIiIGhlaWdodD0iMzIiIHZpZXdCb3g9IjAgMCAzMiAzMiIgYXJpYS1oaWRkZW49InRydWUiIGNsYXNzPSJlY2NndWktaWNvbiI+CiAgPCEtLSBTaGFyZWQgYmFzZTogdGhlIEhldHpuZXIgSCwgaWRlbnRpY2FsIGluIGV2ZXJ5IGljb24gb2YgdGhpcyBwYWNrYWdlLgogICAgICAgQSB0YXNrIGljb24gaXMgdGhpcyBwYXRoIHBsdXMgb25lIG1vZGlmaWVyIGluIHRoZSBsb3dlciByaWdodCAxOC4uMzEKICAgICAgIHNxdWFyZS4gS2VlcCB0aGUgYmFzZSBieXRlIGZvciBieXRlIHdoZW4gYWRkaW5nIGFuIGljb24uIC0tPgogIDxwYXRoIGQ9Ik0yLDNoNHY3aDZWM2g0VjIxaC00VjE0SDZ2N0gyWiI+PC9wYXRoPgogIDwhLS0gTW9kaWZpZXI6IGEgbGlzdCwgZm9yIHRoZSBjYXRhbG9nIG9mIG9mZmVyZWQgdHlwZXMuIC0tPgogIDxwYXRoIGQ9Ik0xOCwxOGgxM3YzSDE4WiI+PC9wYXRoPgogIDxwYXRoIGQ9Ik0xOCwyM2gxM3YzSDE4WiI+PC9wYXRoPgogIDxwYXRoIGQ9Ik0xOCwyOGgxM3YzSDE4WiI+PC9wYXRoPgo8L3N2Zz4K", "properties": { - "graphql_url": { - "name": "graphql_url", - "title": "Endpoint", - "description": "The URL of the GraphQL endpoint you want to query. A collective list of public GraphQL APIs is available [here](https://github.com/IvanGoncharov/graphql-apis). Example Endpoint: `https://fruits-api.netlify.app/graphql`", + "api_token": { + "name": "api_token", + "title": "API Token", + "description": "A Hetzner Cloud project API token (read-only is sufficient).", "type": "string", - "parameterType": "string", + "parameterType": "password", "value": null, "advanced": false, "visibleInDialog": true, "properties": {} - }, - "graphql_query": { - "name": "graphql_query", - "title": "Query", - "description": "The query text of the GraphQL Query you want to execute. GraphQL is a query language for APIs and a runtime for fulfilling those queries with your existing data. Learn more on GraphQL [here](https://graphql.org/). Example Query: query allFruits { fruits { id scientific_name tree_name fruit_name family origin description climatic_zone } }", + } + }, + "properties_advanced": { + "architecture": { + "name": "architecture", + "title": "Architecture", + "description": "Restrict to a CPU architecture. Empty lists any architecture.", "type": "string", - "parameterType": "multiline string", - "value": null, - "advanced": false, + "parameterType": "string", + "value": "", + "advanced": true, "visibleInDialog": true, "properties": {} }, - "graphql_variable_values": { - "name": "graphql_variable_values", - "title": "Query variables", - "description": "Pass dynamic variables when making a query or mutation. Example Variables: {\"id\" : 1}", + "cpu_type": { + "name": "cpu_type", + "title": "CPU Type", + "description": "Restrict to a CPU type. Empty lists any CPU type.", "type": "string", - "parameterType": "multiline string", - "value": "{}", - "advanced": false, + "parameterType": "string", + "value": "", + "advanced": true, "visibleInDialog": true, "properties": {} - } - }, - "properties_advanced": { - "graphql_dataset": { - "name": "graphql_dataset", - "title": "Target JSON Dataset", - "description": "The Dataset where this task will save the JSON results.", + }, + "location": { + "name": "location", + "title": "Location", + "description": "Restrict to a single location name (e.g. fsn1). Empty lists all.", "type": "string", "parameterType": "string", "value": "", @@ -3440,13 +4284,24 @@ "visibleInDialog": true, "properties": {} }, - "oauth_access_token": { - "name": "oauth_access_token", - "title": "OAuth access token", - "description": "Access token that connects to a GraphQL endpoint to authorize and secure user access to resources and data.", + "only_available": { + "name": "only_available", + "title": "Only available", + "description": "If enabled, list only offerings currently available in their location.", "type": "string", - "parameterType": "string", - "value": "", + "parameterType": "boolean", + "value": "false", + "advanced": true, + "visibleInDialog": true, + "properties": {} + }, + "exclude_deprecated": { + "name": "exclude_deprecated", + "title": "Exclude deprecated", + "description": "If enabled, drop offerings that are deprecated in their location.", + "type": "string", + "parameterType": "boolean", + "value": "false", "advanced": true, "visibleInDialog": true, "properties": {} @@ -3454,8 +4309,7 @@ }, "actions": {}, "required": [ - "graphql_url", - "graphql_query" + "api_token" ], "distanceMeasureRange": null, "backendType": "python", @@ -3607,7 +4461,7 @@ ], "main_category": "Uncategorized", "description": "Process a JSON document with a jq filter / program.", - "markdownDocumentation": "\n> [jq](https://jqlang.org/) is like sed for JSON data - you can use it to\n> slice and filter and map and transform structured data with the same ease that sed, awk,\n> grep and friends let you play with text.\n\nIn order to test jq expressions, you can use [play.jqlang.org](https://play.jqlang.org/).\n\n## Basic concepts:\n\n- Filters separated by a comma will produce multiple independent outputs: `,`\n- Will ignores error if the type is unexpected: `?`\n- Array construction: `[]`\n- Object construction: `{}`\n- Concatenate or Add: `+`\n- Difference of sets or Substract: `-`\n- Size of selected element: `length`\n- Pipes are used to chain commands in a similar fashion than bash: `|`\n\n## Dealing with JSON objects\n\n- Display all keys: `keys`\n- Adds + 1 to all items: `map_values(.+1)`\n- Delete a key: `del(.foo)`\n- Convert an object to array: `to_entries | map([.key, .value])`\n\n## Dealing with fields\n\n- Concatenate two fields: `fieldNew=.field1+' '+.field2`\n\n## Dealing with arrays: Slicing and Filtering\n\n- All: `.[]`\n- First: `.[0]`\n- Range: `.[2:4]`\n- First 3: `.[:3]`\n- Last 2: `.[-2:]`\n- Before Last: `.[-2]`\n- Select array of int by value: `map(select(. >= 2))`\n- Select array of objects by value: `.[] | select(.id == \"second\")`\n- Select by type: `.[] | numbers`\n\nTypes can be `arrays`, `objects`, `iterables`, `booleans`, `numbers`, `normals`,\n `finites`, `strings`, `nulls`, `values` and `scalars`.\n\n## Dealing with arrays: Mapping and Transforming\n\n- Add + 1 to all items: `map(.+1)`\n- Delete 2 items: `del(.[1, 2])`\n- Concatenate arrays: `add`\n- Flatten an array: `flatten`\n- Create a range of numbers: `[range(2;4)]`\n- Display the type of each item: `map(type)`\n- Sort an array of basic type: `sort`\n- Sort an array of objects: `sort_by(.foo)`\n- Group by a key - opposite to flatten: `group_by(.foo)`\n- Minimum value of an array: `min` (see also `max`, `min_by(path_exp)`, `max_by(path_exp)`)\n- Remove duplicates: `unique` or `unique_by(.foo)` or `unique_by(length)`\n- Reverse an array: `reverse`\n\n", + "markdownDocumentation": "\nThis task applies a jq filter to a JSON file and writes the transformed\nresult to a new JSON file.\n\nBoth the input and the output port carry a single file per entity. The input\nfile is read as JSON, transparently decompressing it first if it is\ngzip-compressed. The output file is always plain (uncompressed) JSON, encoded\nas UTF-8 and pretty-printed with two-space indentation.\n\nThe task is typically placed between two file-based tasks, for example\nreading from a **JSON dataset** and writing to another one, so that the\nfilter output becomes the next task's input.\n\nIf the filter produces exactly one JSON value for a file, that value becomes\nthe file's entire content. If it produces zero or several values, the output\nfile instead contains a JSON array of all of them - so the shape of the\noutput can change from a single object to an array depending on how many\nvalues the filter happens to emit. The whole input file is read into memory\nbefore the filter runs, so very large files are not streamed. Cancelling the\ncontaining workflow only takes effect between files, not while a file is\nbeing transformed.\n\n> [jq](https://jqlang.org/) is like sed for JSON data - you can use it to\n> slice and filter and map and transform structured data with the same ease that sed, awk,\n> grep and friends let you play with text.\n\nIn order to test jq expressions, you can use [play.jqlang.org](https://play.jqlang.org/).\n\n## Basic concepts:\n\n- Filters separated by a comma will produce multiple independent outputs: `,`\n- Will ignores error if the type is unexpected: `?`\n- Array construction: `[]`\n- Object construction: `{}`\n- Concatenate or Add: `+`\n- Difference of sets or Substract: `-`\n- Size of selected element: `length`\n- Pipes are used to chain commands in a similar fashion than bash: `|`\n\n## Dealing with JSON objects\n\n- Display all keys: `keys`\n- Adds + 1 to all items: `map_values(.+1)`\n- Delete a key: `del(.foo)`\n- Convert an object to array: `to_entries | map([.key, .value])`\n\n## Dealing with fields\n\n- Concatenate two fields: `fieldNew=.field1+' '+.field2`\n\n## Dealing with arrays: Slicing and Filtering\n\n- All: `.[]`\n- First: `.[0]`\n- Range: `.[2:4]`\n- First 3: `.[:3]`\n- Last 2: `.[-2:]`\n- Before Last: `.[-2]`\n- Select array of int by value: `map(select(. >= 2))`\n- Select array of objects by value: `.[] | select(.id == \"second\")`\n- Select by type: `.[] | numbers`\n\nTypes can be `arrays`, `objects`, `iterables`, `booleans`, `numbers`, `normals`,\n `finites`, `strings`, `nulls`, `values` and `scalars`.\n\n## Dealing with arrays: Mapping and Transforming\n\n- Add + 1 to all items: `map(.+1)`\n- Delete 2 items: `del(.[1, 2])`\n- Concatenate arrays: `add`\n- Flatten an array: `flatten`\n- Create a range of numbers: `[range(2;4)]`\n- Display the type of each item: `map(type)`\n- Sort an array of basic type: `sort`\n- Sort an array of objects: `sort_by(.foo)`\n- Group by a key - opposite to flatten: `group_by(.foo)`\n- Minimum value of an array: `min` (see also `max`, `min_by(path_exp)`, `max_by(path_exp)`)\n- Remove duplicates: `unique` or `unique_by(.foo)` or `unique_by(length)`\n- Reverse an array: `reverse`\n\n", "pluginIcon": "data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiIHN0YW5kYWxvbmU9Im5vIj8+CjxzdmcKICAgd2lkdGg9IjUwMCIKICAgaGVpZ2h0PSI1MDAiCiAgIHZlcnNpb249IjEuMCIKICAgdmlld0JveD0iMCAwIDUwMCA1MDAiCiAgIGlkPSJzdmc4NDAiCiAgIHNvZGlwb2RpOmRvY25hbWU9ImpxLnN2ZyIKICAgaW5rc2NhcGU6dmVyc2lvbj0iMS4xLjIgKGI4ZTI1YmU4LCAyMDIyLTAyLTA1KSIKICAgeG1sbnM6aW5rc2NhcGU9Imh0dHA6Ly93d3cuaW5rc2NhcGUub3JnL25hbWVzcGFjZXMvaW5rc2NhcGUiCiAgIHhtbG5zOnNvZGlwb2RpPSJodHRwOi8vc29kaXBvZGkuc291cmNlZm9yZ2UubmV0L0RURC9zb2RpcG9kaS0wLmR0ZCIKICAgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIgogICB4bWxuczpzdmc9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KICA8ZGVmcwogICAgIGlkPSJkZWZzODQ0IiAvPgogIDxzb2RpcG9kaTpuYW1lZHZpZXcKICAgICBpZD0ibmFtZWR2aWV3ODQyIgogICAgIHBhZ2Vjb2xvcj0iI2ZmZmZmZiIKICAgICBib3JkZXJjb2xvcj0iIzY2NjY2NiIKICAgICBib3JkZXJvcGFjaXR5PSIxLjAiCiAgICAgaW5rc2NhcGU6cGFnZXNoYWRvdz0iMiIKICAgICBpbmtzY2FwZTpwYWdlb3BhY2l0eT0iMC4wIgogICAgIGlua3NjYXBlOnBhZ2VjaGVja2VyYm9hcmQ9IjAiCiAgICAgc2hvd2dyaWQ9ImZhbHNlIgogICAgIGlua3NjYXBlOnpvb209IjAuNzI3MDAyNTMiCiAgICAgaW5rc2NhcGU6Y3g9IjIzNC41MjQ2MyIKICAgICBpbmtzY2FwZTpjeT0iMjg2Ljc5NDA1IgogICAgIGlua3NjYXBlOndpbmRvdy13aWR0aD0iMTkyMCIKICAgICBpbmtzY2FwZTp3aW5kb3ctaGVpZ2h0PSIxMDI3IgogICAgIGlua3NjYXBlOndpbmRvdy14PSIxNzI4IgogICAgIGlua3NjYXBlOndpbmRvdy15PSIyNSIKICAgICBpbmtzY2FwZTp3aW5kb3ctbWF4aW1pemVkPSIwIgogICAgIGlua3NjYXBlOmN1cnJlbnQtbGF5ZXI9InN2Zzg0MCIgLz4KICA8ZwogICAgIGlkPSJnOTIwIgogICAgIHRyYW5zZm9ybT0idHJhbnNsYXRlKDQ4LjE0Mjg4NiwxMzAuNjczNTUpIj4KICAgIDxjaXJjbGUKICAgICAgIGN4PSIyMiIKICAgICAgIGN5PSIxNTciCiAgICAgICByPSIxOSIKICAgICAgIGlkPSJjaXJjbGU4MjYiCiAgICAgICBzdHlsZT0iZmlsbC1vcGFjaXR5OjE7c3Ryb2tlLXdpZHRoOjAuMSIgLz4KICAgIDxwYXRoCiAgICAgICBkPSJtIDEzMywzMS4yIGMgLTMsNC4xIC02OS44LDE0OS4yIC03MSwxNTIuOCAtMS4yLDMuNiAyLjUsNi42IDEwLDEwLjEgNy43LDMuNiAxMC4yLDMuNCAxMi40LDAuMyAyLjIsLTMuMSA3MC4xLC0xNDcuNyA3MS42LC0xNTIuNSAxLjUsLTQuNyAtMiwtNy4zIC04LjQsLTEwLjQgQyAxNDEuMiwyOC40IDEzNiwyNy4xIDEzMywzMS4yIFoiCiAgICAgICBpZD0icGF0aDgyOCIKICAgICAgIHN0eWxlPSJmaWxsLW9wYWNpdHk6MTtzdHJva2Utd2lkdGg6MC4xIiAvPgogICAgPGcKICAgICAgIGlkPSJnODM2IgogICAgICAgdHJhbnNmb3JtPSJzY2FsZSgwLjEpIj4KICAgICAgPGNpcmNsZQogICAgICAgICBjeD0iMjQxMCIKICAgICAgICAgY3k9IjIwMCIKICAgICAgICAgcj0iMTkwIgogICAgICAgICBpZD0iY2lyY2xlODMyIiAvPgogICAgICA8cGF0aAogICAgICAgICBkPSJtIDE4MzAsNjU1IGMgMCwxMjkgMCwxMzAgNzAsMTMwIGggMzYwIHYgNDIzIGMgMCwyNDYgLTUsNDc0IC0xMCw1MDYgLTE5LDExMiAtODgsMTc2IC0xOTAsMTc2IC01OSwwIC0xMjAsLTI3IC0xNDcsLTY2IC01MSwtNzMgLTY1LC04NCAtOTksLTg0IC0zMSwwIC00MCw4IC0xMDQsOTMgLTg0LDExMSAtODYsMTI5IC0zMiwxODkgMjI3LDI1MiA2OTIsMTg3IDgzNiwtMTE4IDUzLC0xMTIgNTYsLTE1MyA1NiwtNzk1IFYgNTk1IGMgMCwtNjQgLTMwLC0xMDAgLTEwMCwtMTAwIGggLTQ5MCBjIC0xNDksMCAtMTUwLDAgLTE1MCwxNjAgeiBtIDEwMzcsNTc4IGMgMzAsMzAyIDI1MCw0OTkgNDM4LDUyNyAxNDQsMTkgMjQ0LDAgMzc1LC05MCB2IDQxMCBjIDAsODEgMCwxMDAgMTUwLDEwMCAxMzYsMCAxNTAsLTIwIDE1MCwtMTUwIFYgNjEwIGMgMCwtOTkgLTE5LC0xMDAgLTE1MCwtMTAwIC0xMzEsMCAtMTM5LDE1IC0xMzksODIgLTEwMywtODkgLTM1MSwtMTQ5IC01MjMsLTQ5IC0xNzIsMTAwIC0zNDIsMjc4IC0zMDEsNjkwIHogbSA4MDMsLTEwNyBjIDAsMjkyIC0xMDUsMzU2IC0yMzcsMzU2IC0xNzgsMCAtMjQ5LC0xNDcgLTI3MywtMjk2IC0yNywtMTY4IDI1LC00MjAgMjYyLC00MjAgMTcxLDAgMjQ4LDg5IDI0OCwzNjAgeiIKICAgICAgICAgaWQ9InBhdGg4MzQiIC8+CiAgICA8L2c+CiAgPC9nPgo8L3N2Zz4K", "properties": { "jq_expression": { @@ -3625,8 +4479,8 @@ "properties_advanced": { "validation_source": { "name": "validation_source", - "title": "JSON source which can be used with the validate expression action", - "description": "", + "title": "Validation source", + "description": "Example JSON used by the Validate expression action to preview the filter's output before running the task. Leave empty to skip the preview.", "type": "string", "parameterType": "code-json", "value": "", @@ -3660,8 +4514,8 @@ "Uncategorized" ], "main_category": "Uncategorized", - "description": "Search and retrieve JIRA issues.", - "markdownDocumentation": "This workflow task sends a [JQL query](https://www.atlassian.com/software/jira/guides/jql/overview)\nto the [REST API (v2)](https://developer.atlassian.com/cloud/jira/platform/rest/v2/) of a given\nJira service. It is tested both with on-premise Jira deployments as well as with instances on\n`atlassian.net`.\n\nThe result of the JQL query is a list of JIRA issue descriptions (entities).\nThis list is forwarded as a JSON document to the output port,\nwhere you should connect a JSON Dataset.\n\nNote that you need to create an [API token](https://support.atlassian.com/atlassian-account/docs/manage-api-tokens-for-your-atlassian-account/)\nfor your Atlassian account, to access the API of your atlassian.net hosted Jira instance.\n", + "description": "Search and retrieve Jira issues.", + "markdownDocumentation": "This workflow task sends a [JQL query](https://www.atlassian.com/software/jira/guides/jql/overview)\nto the [REST API (v2)](https://developer.atlassian.com/cloud/jira/platform/rest/v2/) of a given\nJira service. It is tested both with on-premise Jira deployments as well as with instances on\n`atlassian.net`.\n\nThe result of the JQL query is a list of Jira issue descriptions (entities).\nThis list is forwarded as a JSON document to the output port,\nwhere you should connect a JSON Dataset.\n\nNote that you need to create an [API token](https://support.atlassian.com/atlassian-account/docs/manage-api-tokens-for-your-atlassian-account/)\nfor your Atlassian account, to access the API of your atlassian.net hosted Jira instance.\n", "pluginIcon": "data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiIHN0YW5kYWxvbmU9Im5vIj8+CjxzdmcKICAgdmlld0JveD0iMCAwIDI0IDI0IgogICB2ZXJzaW9uPSIxLjEiCiAgIGlkPSJzdmc4MjYiCiAgIHNvZGlwb2RpOmRvY25hbWU9ImppcmEuc3ZnIgogICBpbmtzY2FwZTp2ZXJzaW9uPSIxLjEuMiAoYjhlMjViZTgsIDIwMjItMDItMDUpIgogICB4bWxuczppbmtzY2FwZT0iaHR0cDovL3d3dy5pbmtzY2FwZS5vcmcvbmFtZXNwYWNlcy9pbmtzY2FwZSIKICAgeG1sbnM6c29kaXBvZGk9Imh0dHA6Ly9zb2RpcG9kaS5zb3VyY2Vmb3JnZS5uZXQvRFREL3NvZGlwb2RpLTAuZHRkIgogICB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciCiAgIHhtbG5zOnN2Zz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPgogIDxkZWZzCiAgICAgaWQ9ImRlZnM4MzAiIC8+CiAgPHNvZGlwb2RpOm5hbWVkdmlldwogICAgIGlkPSJuYW1lZHZpZXc4MjgiCiAgICAgcGFnZWNvbG9yPSIjZmZmZmZmIgogICAgIGJvcmRlcmNvbG9yPSIjNjY2NjY2IgogICAgIGJvcmRlcm9wYWNpdHk9IjEuMCIKICAgICBpbmtzY2FwZTpwYWdlc2hhZG93PSIyIgogICAgIGlua3NjYXBlOnBhZ2VvcGFjaXR5PSIwLjAiCiAgICAgaW5rc2NhcGU6cGFnZWNoZWNrZXJib2FyZD0iMCIKICAgICBzaG93Z3JpZD0iZmFsc2UiCiAgICAgaW5rc2NhcGU6em9vbT0iMzUuMjkxNjY3IgogICAgIGlua3NjYXBlOmN4PSIxMiIKICAgICBpbmtzY2FwZTpjeT0iMTIiCiAgICAgaW5rc2NhcGU6d2luZG93LXdpZHRoPSIxOTIwIgogICAgIGlua3NjYXBlOndpbmRvdy1oZWlnaHQ9IjEwMjciCiAgICAgaW5rc2NhcGU6d2luZG93LXg9IjE3MjgiCiAgICAgaW5rc2NhcGU6d2luZG93LXk9IjI1IgogICAgIGlua3NjYXBlOndpbmRvdy1tYXhpbWl6ZWQ9IjEiCiAgICAgaW5rc2NhcGU6Y3VycmVudC1sYXllcj0ic3ZnODI2IiAvPgogIDxwYXRoCiAgICAgZD0ibSAxMS42MjE4OTIsMy45NTUxMzU3IGMgMCwxLjkzMDc2NzUgMS41ODQ4MzgsMy40OTk1MTYgMy40OTk1MTUsMy40OTk1MTYgaCAxLjQzMTk4NiB2IDEuMzY3NjI3IGMgMCwxLjkzMDc2NzMgMS41NjA3MDQsMy40OTE0NzAzIDMuNDkxNDcxLDMuNDk5NTE2MyBWIDQuNjMwOTA0NCBBIDAuNjc1NzY4NTksMC42NzU3Njg1OSAwIDAgMCAxOS4zNjkwOTYsMy45NTUxMzU3IFogTSA3Ljc5MjUzNiw3LjgxNjY3MDYgYSAzLjUwNzU2MDgsMy41MDc1NjA4IDAgMCAwIDMuNDkxNDcxLDMuNDkxNDcwNCBoIDEuNDQ4MDc2IHYgMS4zODM3MTggYSAzLjUwNzU2MDgsMy41MDc1NjA4IDAgMCAwIDMuNDkxNDcsMy40OTE0NyBWIDguNDg0Mzk0MyBBIDAuNjc1NzY4NTksMC42NzU3Njg1OSAwIDAgMCAxNS41NTU4Myw3LjgxNjY3MDYgWiBNIDMuOTU1MTM1NywxMS42NzgyMDUgYyAwLDEuOTMwNzY4IDEuNTY4NzQ4NiwzLjQ5MTQ3MiAzLjQ5OTUxNiwzLjQ5MTQ3MiBoIDEuNDMxOTg1OSB2IDEuMzgzNzE2IGMgMC4wMDgwNSwxLjkyMjcyMyAxLjU2ODc0ODQsMy40OTE0NzEgMy40OTE0NzA0LDMuNDkxNDcxIFYgMTIuMzQ1OTI5IEEgMC42NzU3Njg1OSwwLjY3NTc2ODU5IDAgMCAwIDExLjcwMjM0LDExLjY3MDE2IFoiCiAgICAgaWQ9InBhdGg4MjQiCiAgICAgc3R5bGU9InN0cm9rZS13aWR0aDowLjgwNDQ4NiIgLz4KPC9zdmc+Cg==", "properties": { "base_url": { @@ -3724,7 +4578,7 @@ "ssl_verify": { "name": "ssl_verify", "title": "Verify SSL Connection", - "description": "", + "description": "If enabled, the Jira server's SSL certificate is verified. Disable only when connecting to a server with a self-signed or otherwise untrusted certificate.", "type": "string", "parameterType": "boolean", "value": "true", @@ -3758,7 +4612,7 @@ "actions": { "validate_connection": { "label": "Validate connection", - "description": "Send a minimal test query to JIRA server to validate the connection.", + "description": "Send a minimal test query to the Jira server to validate the connection.", "icon": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTIxLjQgNy41Yy44LjguOCAyLjEgMCAyLjhsLTIuOCAyLjgtNy44LTcuOCAyLjgtMi44Yy44LS44IDIuMS0uOCAyLjggMGwxLjggMS44IDMtMyAxLjQgMS40LTMgM3ptLTUuOCA1LjgtMS40LTEuNC0yLjggMi44LTIuMS0yLjEgMi44LTIuOC0xLjQtMS40LTIuOCAyLjgtMS41LTEuNC0yLjggMi44Yy0uOC44LS44IDIuMSAwIDIuOGwxLjggMS44LTQgNCAxLjQgMS40IDQtNCAxLjggMS44Yy44LjggMi4xLjggMi44IDBsMi44LTIuOC0xLjQtMS40eiIvPjwvc3ZnPg==" } }, @@ -3853,8 +4707,8 @@ "Uncategorized" ], "main_category": "Uncategorized", - "description": "Reads messages from a Kafka topic and saves it to a messages dataset (Consumer).", - "markdownDocumentation": "\nThis workflow operator uses the Kafka Consumer API\nto receive messages from an [Apache Kafka](https://kafka.apache.org/) topic.\n\nMessages received from the topic will be generated as entities with the following\nflat schema:\n\n- **key** - the optional key of the message,\n- **content** - the message itself as plain text (use other operators, such as\n [Parse JSON](JsonParserOperator.md) or [Parse XML](XmlParserOperator.md) to process\n complex message content),\n- **offset** - the given offset of the message in the topic,\n- **ts-production** - the timestamp when the message was written to the topic,\n- **ts-consumption** - the timestamp when the message was consumed from the topic.\n\nIn order to process the resulting entities, they have to run through a transformation.\n\nAs an alternate working mode, messages can be exported directly to a JSON or XML\ndataset if you know that the messages on your topic are valid JSON or XML documents\n(see Advanced Options > Messages Dataset).\n\nIn this case, a sample response from the consumer will appear as follows:\n\n
\n Sample JSON Response\n\n```json\n[\n {\n \"message\": {\n \"key\": \"818432-942813-832642-453478\",\n \"headers\": {\n \"type\": \"ADD\"\n },\n \"content\": {\n \"location\": [\"Leipzig\"],\n \"obstacle\": {\n \"name\": \"Iron Bars\",\n \"order\": \"1\"\n }\n }\n }\n },\n {\n \"message\": {\n \"key\": \"887428-119918-570674-866526\",\n \"headers\": {\n \"type\": \"REMOVE\"\n },\n \"content\": {\n \"comments\": \"We can pass any json payload here.\"\n }\n }\n },\n {\n \"message\": {\n \"key\": \"TestKey\",\n \"tombstone\": true,\n \"headers\": {\n \"h1\": \"v1\",\n \"h2\": \"v2\"\n },\n \"content\": {\n \"will_be_ignored\": \"...\"\n }\n }\n }\n]\n```\n\n
\n
\n Sample XML Response\n\n```xml\n \n \n \n \n \n string\n \n \n \n \n \n \n string\n \n \n \n will be ignored\n \n```\n\n
\n", + "description": "Reads messages from a Kafka topic as entities, or saves them to a dataset (Consumer).", + "markdownDocumentation": "\nThis workflow operator uses the Kafka Consumer API\nto receive messages from an [Apache Kafka](https://kafka.apache.org/) topic. It has\nno input port: it starts a workflow by reading from the topic on its own.\n\nBy default, each message received is generated as one entity with the following\nflat schema:\n\n- **key** - the optional key of the message,\n- **content** - the message itself as plain text (use other operators, such as\n [Parse JSON](JsonParserOperator.md) or [Parse XML](XmlParserOperator.md) to process\n complex message content),\n- **offset** - the given offset of the message in the topic,\n- **ts-production** - the timestamp when the message was written to the topic,\n- **ts-consumption** - the timestamp when the message was consumed from the topic.\n\nIn order to process the resulting entities, they have to run through a transformation.\n\nAs an alternate working mode, messages can be exported directly to a JSON or XML\ndataset if you know that the messages on your topic are valid JSON or XML documents\n(see the Messages Dataset parameter). In this mode the task has no output port either\n- messages are written directly to the dataset instead of being handed on as entities.\n\nIn this case, a sample response from the consumer will appear as follows:\n\n
\n Sample JSON Response\n\n```json\n[\n {\n \"message\": {\n \"key\": \"818432-942813-832642-453478\",\n \"headers\": {\n \"type\": \"ADD\"\n },\n \"content\": {\n \"location\": [\"Leipzig\"],\n \"obstacle\": {\n \"name\": \"Iron Bars\",\n \"order\": \"1\"\n }\n }\n }\n },\n {\n \"message\": {\n \"key\": \"887428-119918-570674-866526\",\n \"headers\": {\n \"type\": \"REMOVE\"\n },\n \"content\": {\n \"comments\": \"We can pass any json payload here.\"\n }\n }\n },\n {\n \"message\": {\n \"key\": \"TestKey\",\n \"tombstone\": true,\n \"headers\": {\n \"h1\": \"v1\",\n \"h2\": \"v2\"\n },\n \"content\": {\n \"will_be_ignored\": \"...\"\n }\n }\n }\n]\n```\n\n
\n
\n Sample XML Response\n\n```xml\n \n \n \n \n \n string\n \n \n \n \n \n \n string\n \n \n \n will be ignored\n \n```\n\n
\n", "pluginIcon": null, "properties": { "message_dataset": { @@ -3994,7 +4848,7 @@ "disable_commit": { "name": "disable_commit", "title": "Disable Commit", - "description": "Setting this to true will disable committing messages after retrival. This means you will receive the same messages on the next execution (for debugging).", + "description": "Setting this to true will disable committing messages after retrieval. This means you will receive the same messages on the next execution (for debugging).", "type": "string", "parameterType": "boolean", "value": "false", @@ -4025,14 +4879,14 @@ "Uncategorized" ], "main_category": "Uncategorized", - "description": "Reads a messages dataset and sends records to a Kafka topic (Producer).", - "markdownDocumentation": "This workflow operator uses the Kafka Producer API to send\nmessages to a [Apache Kafka](https://kafka.apache.org/).\n\nAccepts entities as input, and, if desired, accepts a pre-constructed XML/JSON dataset,\nwhich is transformed into messages and sent to a designated Kafka topic based\non configuration.\n\n
\n Sample XML format\n\n An example XML document is shown below. This document will be sent as two messages\n to the configured topic. Each message is created as a proper XML document.\n\n\n```xml\n \n \n \n \n \n string\n \n \n \n \n \n \n string\n \n \n \n will be ignored\n \n```\n\n
\n\n
\n Sample JSON format\n\n An example JSON document is shown below. This document will be sent as two messages\n to the configured topic. Each message is created as a proper JSON document.\n\n\n```json\n[\n {\n \"message\": {\n \"key\": \"818432-942813-832642-453478\",\n \"headers\": {\n \"type\": \"ADD\"\n },\n \"content\": {\n \"location\": [\"Leipzig\"],\n \"obstacle\": {\n \"name\": \"Iron Bars\",\n \"order\": \"1\"\n }\n }\n }\n },\n {\n \"message\": {\n \"key\": \"887428-119918-570674-866526\",\n \"headers\": {\n \"type\": \"REMOVE\"\n },\n \"content\": {\n \"comments\": \"We can pass any json payload here.\"\n }\n }\n },\n {\n \"message\": {\n \"key\": \"TestKey\",\n \"tombstone\": true,\n \"headers\": {\n \"h1\": \"v1\",\n \"h2\": \"v2\"\n },\n \"content\": {\n \"will_be_ignored\": \"...\"\n }\n }\n }\n]\n```\n\n
\n\n", + "description": "Sends entities, or messages from a dataset, to a Kafka topic (Producer).", + "markdownDocumentation": "This workflow operator uses the Kafka Producer API to send\nmessages to an [Apache Kafka](https://kafka.apache.org/) topic.\n\nBy default, each input entity is sent as one message, serialized as JSON with the\nentity's URI, schema type and property values; the message key is always empty in\nthis mode. If a Messages Dataset is configured instead, the input port is replaced\nby that dataset, and messages are read from a pre-constructed XML or JSON document\ninstead of the workflow's entities. This task has no output port: it is a terminal\nstep and hands nothing on to further operators.\n\n
\n Sample XML format\n\n An example XML document is shown below. This document will be sent as two messages\n to the configured topic. Each message is created as a proper XML document. A\n `` element with more than one child cannot be turned into a single XML\n document; it is skipped, and an error is logged.\n\n\n```xml\n \n \n \n \n \n string\n \n \n \n \n \n \n string\n \n \n \n will be ignored\n \n```\n\n
\n\n
\n Sample JSON format\n\n An example JSON document is shown below. This document will be sent as two messages\n to the configured topic. Each message is created as a proper JSON document.\n\n\n```json\n[\n {\n \"message\": {\n \"key\": \"818432-942813-832642-453478\",\n \"headers\": {\n \"type\": \"ADD\"\n },\n \"content\": {\n \"location\": [\"Leipzig\"],\n \"obstacle\": {\n \"name\": \"Iron Bars\",\n \"order\": \"1\"\n }\n }\n }\n },\n {\n \"message\": {\n \"key\": \"887428-119918-570674-866526\",\n \"headers\": {\n \"type\": \"REMOVE\"\n },\n \"content\": {\n \"comments\": \"We can pass any json payload here.\"\n }\n }\n },\n {\n \"message\": {\n \"key\": \"TestKey\",\n \"tombstone\": true,\n \"headers\": {\n \"h1\": \"v1\",\n \"h2\": \"v2\"\n },\n \"content\": {\n \"will_be_ignored\": \"...\"\n }\n }\n }\n]\n```\n\n
\n\n", "pluginIcon": null, "properties": { "message_dataset": { "name": "message_dataset", "title": "Messages Dataset", - "description": "Where do you want to retrieve the messages from? The dropdown lists usable datasets from the current project only. In case you miss your dataset, check for the correct type (XML/JSON) and build project). The messages will be retrieved from the entities if no dataset is provided.", + "description": "Where do you want to retrieve the messages from? The dropdown lists usable datasets from the current project only. In case you miss your dataset, check for the correct type (XML/JSON) and build project. The messages will be retrieved from the entities if no dataset is provided.", "type": "string", "parameterType": "string", "value": "", @@ -4157,6 +5011,59 @@ "pluginType": "customtask", "relatedPlugins": [] }, + "cmem_plugin_querycatalog-ListCatalogQueries": { + "pluginId": "cmem_plugin_querycatalog-ListCatalogQueries", + "title": "List Catalog Queries", + "categories": [ + "Uncategorized" + ], + "main_category": "Uncategorized", + "description": "Emit one entity per query stored in a query catalog.", + "markdownDocumentation": "\nThis task reads a query catalog and emits one entity for each query it holds,\ndescribing the query rather than running it.\n\nEach entity carries the catalog graph, the query IRI, its label and description, its\ntype, its query text as stored, and the names of the placeholders that text contains.\nThe query text is emitted unfilled, with its placeholders intact. The task takes no\ninput, so it can open a workflow.\n\nIt emits exactly what **Get Catalog Query** emits, in the same order, so one can\nreplace the other without rewiring whatever consumes it. Use this task to take stock of\na catalog, to drive a loop over its queries, or to find the query IRI that\n**Execute Catalog Query** should be pointed at.\n\nA catalog holding no query of the selected kind yields no entities and no error, which\nis a legitimate answer about an empty catalog rather than a failure.\n", + "pluginIcon": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAzMiAzMiIgZmlsbD0iY3VycmVudENvbG9yIj4KICA8Y2lyY2xlIGN4PSIyMC4zNDUiIGN5PSIyMS45MjUiIHI9IjEuNTgiLz4KICA8cGF0aCBkPSJNMjYuNDg4OCwyMS41MTNBNi44MjU2LDYuODI1NiwwLDAsMCwyMC4zNDUsMTcuMTg1YTYuODI1Niw2LjgyNTYsMCwwLDAsLTYuMTQzOCw0LjMyOEwxNC4wMjUsMjEuOTI1bDAuMTc2MiwwLjQxMkE2LjgyNTYsNi44MjU2LDAsMCwwLDIwLjM0NSwyNi42NjVhNi44MjU2LDYuODI1NiwwLDAsMCw2LjE0MzgsLTQuMzI4TDI2LjY2NSwyMS45MjVaTTIwLjM0NSwyNS4wODVhMy4xNiwzLjE2LDAsMSwxLDMuMTYsLTMuMTZBMy4xNjM2LDMuMTYzNiwwLDAsMSwyMC4zNDUsMjUuMDg1WiIvPgogIDxjaXJjbGUgY3g9IjkuMjg1IiBjeT0iOS4yODUiIHI9IjAuNzkiLz4KICA8Y2lyY2xlIGN4PSI5LjI4NSIgY3k9IjE1LjYwNSIgcj0iMC43OSIvPgogIDxjaXJjbGUgY3g9IjkuMjg1IiBjeT0iMjEuOTI1IiByPSIwLjc5Ii8+CiAgPHBhdGggZD0iTTYuOTE1LDE5LjU1NWg1LjUzVjE3Ljk3NUg2LjkxNVYxMy4yMzVIMTkuNTU1djIuMzdoMS41OFY2LjkxNWExLjU4LDEuNTgsMCwwLDAsLTEuNTgsLTEuNThINi45MTVBMS41OCwxLjU4LDAsMCwwLDUuMzM1LDYuOTE1VjI0LjI5NWExLjU4LDEuNTgsMCwwLDAsMS41OCwxLjU4aDUuNTNWMjQuMjk1SDYuOTE1Wk02LjkxNSw2LjkxNUgxOS41NTV2NC43NEg2LjkxNVoiLz4KPC9zdmc+Cg==", + "properties": { + "catalog_graph": { + "name": "catalog_graph", + "title": "Catalog Graph", + "description": "The graph the query is read from. The list offers the query catalogs first, then the shape catalogs, then any other graph that holds queries, with the number of read and write queries each one contains.", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "query_kind": { + "name": "query_kind", + "title": "Query Kind", + "description": "Which half of the catalog the query list offers.", + "type": "string", + "parameterType": "string", + "value": "all", + "advanced": false, + "visibleInDialog": true, + "properties": {} + } + }, + "properties_advanced": {}, + "actions": { + "preview": { + "label": "Preview", + "description": "List the first queries this task would emit.", + "icon": null + } + }, + "required": [], + "distanceMeasureRange": null, + "backendType": "python", + "is_deprecated": false, + "tags": [ + "WorkflowTask", + "PythonPlugin" + ], + "pluginType": "customtask", + "relatedPlugins": [] + }, "cmem_plugin_git-List": { "pluginId": "cmem_plugin_git-List", "title": "List Git files", @@ -4479,84 +5386,264 @@ "visibleInDialog": true, "properties": {} }, - "no_subfolder": { - "name": "no_subfolder", - "title": "Exclude files in subfolders", - "description": "A flag indicating if files should only be listed from subfolders or not.", + "no_subfolder": { + "name": "no_subfolder", + "title": "Exclude files in subfolders", + "description": "A flag indicating if files should only be listed from subfolders or not.", + "type": "string", + "parameterType": "boolean", + "value": "false", + "advanced": false, + "visibleInDialog": true, + "properties": {} + } + }, + "properties_advanced": { + "max_workers": { + "name": "max_workers", + "title": "Maximum amount of workers", + "description": "Specifies the maximum number of threads used for parallel execution of the workflow. The default is 32, and the valid range is 1 to 32. Note: Due to known throttling limits imposed by Microsoft, running with high parallelism may cause errors. If you encounter issues, try reducing the number of threads to 1.", + "type": "string", + "parameterType": "Long", + "value": "32", + "advanced": true, + "visibleInDialog": true, + "properties": {} + } + }, + "actions": { + "preview_results": { + "label": "Preview results", + "description": "Preview the results.", + "icon": null + } + }, + "required": [ + "tenant_id", + "client_id", + "client_secret", + "type_resource", + "target_resource", + "drives" + ], + "distanceMeasureRange": null, + "backendType": "python", + "is_deprecated": false, + "tags": [ + "WorkflowTask", + "PythonPlugin" + ], + "pluginType": "customtask", + "relatedPlugins": [] + }, + "cmem_plugin_project_resources-List": { + "pluginId": "cmem_plugin_project_resources-List", + "title": "List project files", + "categories": [ + "Uncategorized" + ], + "main_category": "Uncategorized", + "description": "List file resources from the project.", + "markdownDocumentation": "List file resources from the current project based on a regular expression.\n\nThe project-relative path of each file of the current project is tested against a given\nregular expression.\nThe project resource is listed in the output, if the expression matches this path.\nThe output entities have the following paths:\n\n- `name` - the plain file name of the resource (example: `file.txt`)\n- `fullPath` - the project-relative path including directories but no leading slash\n (example: `directory/file.txt`)\n- `modified` - modified timestamp (example: `2025-03-10T15:38:41.023Z`)\n- `size` - size of the file in bytes (example: `123345`)\n\nThe regular expression has to match the `fullPath` of the file and is case sensitive.\n\nGiven this list of example files of a project:\n\n```\ndataset.csv\nmy-dataset.xml\njson/example.json\njson/example_new.json\njson/data.xml\n```\n\nHere are some regular expressions with the expected result:\n\n- The regex `dataset\\.csv` lists only the first file.\n- The regex `json/.*` lists all files in the `json` sub-directory.\n- The regex `new` lists nothing.\n- The regex `.*new.*` list the file `json/example_new.json`\n(and all other files with `new` in the path)\n\nWe recommend to test your regular expression before using it.\n[regex101.com](https://regex101.com) is a proper service to test your regular expressions.\n[This deep-link](https://regex101.com/?testString=dataset.csv%0Amy-dataset.xml%0Ajson/example.json%0Ajson/example_new.json%0Ajson/data.xml®ex=.*new.*)\nprovides a test bed using the example files and the last expression from the list.\n", + "pluginIcon": "data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiPz4KPHN2ZyBpZD0iaWNvbiIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiB3aWR0aD0iMzIiIGhlaWdodD0iMzIiIHZpZXdCb3g9IjAgMCAzMiAzMiI+CiAgPGRlZnM+CiAgICA8c3R5bGU+CiAgICAgIC5jbHMtMSB7CiAgICAgICAgZmlsbDogbm9uZTsKICAgICAgfQogICAgPC9zdHlsZT4KICA8L2RlZnM+CiAgPHBhdGggZD0iTTEzLDI2SDRWNmg3LjJsMy40LDMuNC42LjZoMTIuOHY4aDJ2LThjMC0xLjEtLjktMi0yLTJoLTEybC0zLjQtMy40Yy0uNC0uNC0uOS0uNi0xLjQtLjZoLTcuMmMtMS4xLDAtMiwuOS0yLDJ2MjBjMCwxLjEuOSwyLDIsMmg5czAtMiwwLTJaTTE2LDIwaDE0djJoLTE0di0yWk0xNiwyNGgxNHYyaC0xNHYtMlpNMTYsMjhoN3YyaC03di0yWiIvPgogIDxyZWN0IGlkPSJfeDNDX1RyYW5zcGFyZW50X1JlY3RhbmdsZV94M0VfIiBjbGFzcz0iY2xzLTEiIHdpZHRoPSIzMiIgaGVpZ2h0PSIzMiIvPgo8L3N2Zz4=", + "properties": { + "files_regex": { + "name": "files_regex", + "title": "File matching regex", + "description": "The regex for filtering the file names. The regex needs to match the full path (i.e. from beginning to end, including sub-directories) in order for the file to be deleted.", + "type": "string", + "parameterType": "string", + "value": null, + "advanced": false, + "visibleInDialog": true, + "properties": {} + } + }, + "properties_advanced": {}, + "actions": {}, + "required": [ + "files_regex" + ], + "distanceMeasureRange": null, + "backendType": "python", + "is_deprecated": false, + "tags": [ + "WorkflowTask", + "PythonPlugin" + ], + "pluginType": "customtask", + "relatedPlugins": [] + }, + "cmem_plugin_s3-List": { + "pluginId": "cmem_plugin_s3-List", + "title": "List S3 objects", + "categories": [ + "Uncategorized" + ], + "main_category": "Uncategorized", + "description": "List the objects of a bucket in S3 or S3-compatible object storage.", + "markdownDocumentation": "\nLists the objects of a bucket, together with the metadata the service reports for\nthem.\n\n## Ports\n\nThe task takes no input. It emits one entity per object, carrying the `bucket` it\nwas found in, its full `key`, the `name` that key ends in, its `size` in bytes,\nits `etag`, the `last_modified` timestamp and the `storage_class`. Keys ending in\na slash are folder markers rather than content and are left out.\n\n## Where the task sits\n\n**Download S3 objects** accepts these entities on its input port, so a workflow\ncan list first, narrow the result down with transformations in between, and fetch\nonly what is left. The `etag` travels with each entity and is what lets the\ndownload notice an object that changed in the meantime.\n\nListing is also a way to write an inventory of a bucket into a Knowledge Graph\nwithout moving any of its content.\n\n## Caveats\n\n* **The listing streams.** Entities are produced while the service is still being\n paged through, so a bucket with millions of keys does not have to be held in\n memory - but neither is the total known in advance, and the entity count in the\n report grows as the run proceeds.\n* **A regular expression is applied after the fact.** The key prefix is the only\n filter the service itself applies; everything the expression rejects has\n already been listed and paid for.\n* **The preview stops after ten objects.** It therefore never reaches an object\n further down that a real run would reject or fail on.\n* **An object's size and timestamp are the service's.** They are written down as\n reported, not checked against the bytes.\n ", + "pluginIcon": "data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiIHN0YW5kYWxvbmU9Im5vIj8+CjxzdmcgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiB2aWV3Qm94PSIwIDAgMjQgMjQiIGZpbGw9Im5vbmUiIHN0cm9rZT0iY3VycmVudENvbG9yIiBzdHJva2Utd2lkdGg9IjEuNiIgc3Ryb2tlLWxpbmVjYXA9InJvdW5kIiBzdHJva2UtbGluZWpvaW49InJvdW5kIj4KICA8ZWxsaXBzZSBjeD0iMTIiIGN5PSI1IiByeD0iNyIgcnk9IjIuNiIvPgogIDxwYXRoIGQ9Ik01IDV2Ni40YzAgMS40NCAzLjEzIDIuNiA3IDIuNnM3LTEuMTYgNy0yLjZWNSIvPgogIDxwYXRoIGQ9Ik03IDE3LjZoMTAiLz4KICA8cGF0aCBkPSJNNyAyMC44aDEwIi8+Cjwvc3ZnPgo=", + "properties": { + "endpoint_url": { + "name": "endpoint_url", + "title": "Endpoint URL", + "description": "The address of the service, e.g. `https://minio.example.org`. Leave empty for Amazon S3, which is then addressed by region.", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "region": { + "name": "region", + "title": "Region", + "description": "The region the bucket lives in. Amazon S3 needs the bucket's own region; most other implementations accept any value and ignore it.", + "type": "string", + "parameterType": "string", + "value": "us-east-1", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "credentials": { + "name": "credentials", + "title": "Credentials", + "description": "Where the credentials come from.", + "type": "string", + "parameterType": "string", + "value": "explicit", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "access_key_id": { + "name": "access_key_id", + "title": "Access key ID", + "description": "The access key to sign requests with.", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "secret_access_key": { + "name": "secret_access_key", + "title": "Secret access key", + "description": "The secret belonging to the access key.", + "type": "string", + "parameterType": "password", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "bucket": { + "name": "bucket", + "title": "Bucket", + "description": "The bucket whose objects are listed.", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "prefix": { + "name": "prefix", + "title": "Key prefix", + "description": "Lists only the objects whose key starts with this, e.g. `raw/2026/`. The service applies it, so it is the cheapest way to narrow a large bucket down. Empty means the whole bucket.", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "regex": { + "name": "regex", + "title": "Regular expression", + "description": "Lists only the objects whose key matches this regular expression, e.g. `\\.csv$`. It is matched against the whole key rather than against the name alone, and it has to match somewhere in it rather than all of it. Empty means every key.", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "top_level_only": { + "name": "top_level_only", + "title": "Top level only", + "description": "If enabled, only the keys directly below the prefix are considered and everything nested deeper is folded away. The service does the folding, on a `/` delimiter.", + "type": "string", + "parameterType": "boolean", + "value": "false", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "limit": { + "name": "limit", + "title": "Limit", + "description": "The maximum number of objects to list, or 0 for no limit. A run that stops at the limit says so in its report.", "type": "string", - "parameterType": "boolean", - "value": "false", + "parameterType": "Long", + "value": "0", "advanced": false, "visibleInDialog": true, "properties": {} } }, "properties_advanced": { - "max_workers": { - "name": "max_workers", - "title": "Maximum amount of workers", - "description": "Specifies the maximum number of threads used for parallel execution of the workflow. The default is 32, and the valid range is 1 to 32. Note: Due to known throttling limits imposed by Microsoft, running with high parallelism may cause errors. If you encounter issues, try reducing the number of threads to 1.", + "session_token": { + "name": "session_token", + "title": "Session token", + "description": "The session token of a temporary credential. Needed only for keys issued by a token service, which expire.", "type": "string", - "parameterType": "Long", - "value": "32", + "parameterType": "password", + "value": "", "advanced": true, "visibleInDialog": true, "properties": {} - } - }, - "actions": { - "preview_results": { - "label": "Preview results", - "description": "Preview the results.", - "icon": null - } - }, - "required": [ - "tenant_id", - "client_id", - "client_secret", - "type_resource", - "target_resource", - "drives" - ], - "distanceMeasureRange": null, - "backendType": "python", - "is_deprecated": false, - "tags": [ - "WorkflowTask", - "PythonPlugin" - ], - "pluginType": "customtask", - "relatedPlugins": [] - }, - "cmem_plugin_project_resources-List": { - "pluginId": "cmem_plugin_project_resources-List", - "title": "List project files", - "categories": [ - "Uncategorized" - ], - "main_category": "Uncategorized", - "description": "List file resources from the project.", - "markdownDocumentation": "List file resources from the current project based on a regular expression.\n\nThe project-relative path of each file of the current project is tested against a given\nregular expression.\nThe project resource is listed in the output, if the expression matches this path.\nThe output entities have the following paths:\n\n- `name` - the plain file name of the resource (example: `file.txt`)\n- `fullPath` - the project-relative path including directories but no leading slash\n (example: `directory/file.txt`)\n- `modified` - modified timestamp (example: `2025-03-10T15:38:41.023Z`)\n- `size` - size of the file in bytes (example: `123345`)\n\nThe regular expression has to match the `fullPath` of the file and is case sensitive.\n\nGiven this list of example files of a project:\n\n```\ndataset.csv\nmy-dataset.xml\njson/example.json\njson/example_new.json\njson/data.xml\n```\n\nHere are some regular expressions with the expected result:\n\n- The regex `dataset\\.csv` lists only the first file.\n- The regex `json/.*` lists all files in the `json` sub-directory.\n- The regex `new` lists nothing.\n- The regex `.*new.*` list the file `json/example_new.json`\n(and all other files with `new` in the path)\n\nWe recommend to test your regular expression before using it.\n[regex101.com](https://regex101.com) is a proper service to test your regular expressions.\n[This deep-link](https://regex101.com/?testString=dataset.csv%0Amy-dataset.xml%0Ajson/example.json%0Ajson/example_new.json%0Ajson/data.xml®ex=.*new.*)\nprovides a test bed using the example files and the last expression from the list.\n", - "pluginIcon": "data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiPz4KPHN2ZyBpZD0iaWNvbiIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiB3aWR0aD0iMzIiIGhlaWdodD0iMzIiIHZpZXdCb3g9IjAgMCAzMiAzMiI+CiAgPGRlZnM+CiAgICA8c3R5bGU+CiAgICAgIC5jbHMtMSB7CiAgICAgICAgZmlsbDogbm9uZTsKICAgICAgfQogICAgPC9zdHlsZT4KICA8L2RlZnM+CiAgPHBhdGggZD0iTTEzLDI2SDRWNmg3LjJsMy40LDMuNC42LjZoMTIuOHY4aDJ2LThjMC0xLjEtLjktMi0yLTJoLTEybC0zLjQtMy40Yy0uNC0uNC0uOS0uNi0xLjQtLjZoLTcuMmMtMS4xLDAtMiwuOS0yLDJ2MjBjMCwxLjEuOSwyLDIsMmg5czAtMiwwLTJaTTE2LDIwaDE0djJoLTE0di0yWk0xNiwyNGgxNHYyaC0xNHYtMlpNMTYsMjhoN3YyaC03di0yWiIvPgogIDxyZWN0IGlkPSJfeDNDX1RyYW5zcGFyZW50X1JlY3RhbmdsZV94M0VfIiBjbGFzcz0iY2xzLTEiIHdpZHRoPSIzMiIgaGVpZ2h0PSIzMiIvPgo8L3N2Zz4=", - "properties": { - "files_regex": { - "name": "files_regex", - "title": "File matching regex", - "description": "The regex for filtering the file names. The regex needs to match the full path (i.e. from beginning to end, including sub-directories) in order for the file to be deleted.", + }, + "addressing_style": { + "name": "addressing_style", + "title": "Bucket addressing style", + "description": "How the bucket is named in a request.", "type": "string", "parameterType": "string", - "value": null, - "advanced": false, + "value": "auto", + "advanced": true, + "visibleInDialog": true, + "properties": {} + }, + "ssl_verify": { + "name": "ssl_verify", + "title": "Verify TLS connection", + "description": "If enabled, the certificate of the service is verified. Disable only when connecting to a service with a self-signed or otherwise untrusted certificate.", + "type": "string", + "parameterType": "boolean", + "value": "true", + "advanced": true, "visibleInDialog": true, "properties": {} } }, - "properties_advanced": {}, - "actions": {}, - "required": [ - "files_regex" - ], + "actions": { + "test_connection": { + "label": "Test connection", + "description": "Reach the service, say which identity it signs as and whether the bucket can be listed.", + "icon": null + }, + "preview": { + "label": "Preview (max. 10)", + "description": "List the first objects the current configuration selects.", + "icon": null + } + }, + "required": [], "distanceMeasureRange": null, "backendType": "python", "is_deprecated": false, @@ -4574,14 +5661,14 @@ "Uncategorized" ], "main_category": "Uncategorized", - "description": "List files from a given SSH instance.", - "markdownDocumentation": "\nThis workflow task generates structured output from a specified SSH instance.\n\nBy providing the hostname, username, port and authentication method, you can specify the\nfolder from which the data should be extracted.\n\nYou can also define a regular expression to include or exclude specific files.\n\nThere is also an option to prevent files in subfolders from being included.\n\n#### Authentication Methods:\n* **Password:** Only the password will be used for authentication. The private key field is\nignored, even if filled.\n* **Key:** The private key will be used for authentication. If the key is encrypted, the password\nwill be used to decrypt it.\n\n#### Error handling modes:\n* **Ignore:** Ignores the permission rights of files and lists them all. Skips folders when there\nis no correct permission.\n* **Warning:** Warns the user about files that the user has no permission rights to. Lists all files\nand skips folder when there is no correct permission.\n* **Error:** Throws an error when there is a single file or folder with incorrect permission rights.\n\n#### Note:\n* If a connection cannot be established within 20 seconds, a timeout occurs.\n* Currently supported key types are: RSA, ECDSA, Ed25519.\n* Setting the maximum amount of workers to more than 1 may cause a Channel Exception when\nthe amount of files is too large\n ", + "description": "List the files of a directory on an SSH server, with their metadata.", + "markdownDocumentation": "\nLists the files of a directory on an SSH server, together with the metadata the\nserver reports for them.\n\nThe task takes no input. It emits one entity per file, holding the full remote path\nin `file_name`, plus `size`, `uid`, `gid`, `mode`, `atime` and `mtime` exactly as the\nserver reports them: a byte count, numeric owner and group ids, numeric mode bits and\nUnix timestamps. Folders are traversed but never emitted.\n\n**Download SSH files** accepts these entities on its input port, so that listing and\ndownloading can be split into two steps, with transformations in between that narrow\ndown which of the listed files are fetched.\n\nThe **Preview results** action lists the first ten matches without running the\nworkflow. It stops after ten files, so it can miss the unreadable file that would make\na real run behave differently.\n\n#### Caveats\n\n* Files the account cannot read are listed like every other file. The error handling\nonly decides whether they are reported on top of that, or the listing fails.\n* A folder that cannot be read is skipped silently unless the error handling is set to\nError, and the files below it are then missing from the result without further notice.\n* Every file is opened and one byte is read from it to find out whether it is readable,\nwhich makes listing a large directory noticeably slower than a plain directory listing.\n* The task logs in with the configured credentials only, and offers no key of the\nmachine it runs on. The host key of the server in turn is accepted as presented and\nnever checked against a known hosts list.\n* Establishing the connection fails after 20 seconds.\n ", "pluginIcon": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTQuNSA5Yy0uNiAwLTEgLjUtMSAxdjEuNzVjMCAuNS40IDEgMSAxSDd2Ljc1SDMuNVYxNWg0Yy42IDAgMS0uNSAxLTF2LTEuNzVjMC0uNS0uNC0xLTEtMUg1di0uNzVoMy41Vjl6bTYgMGMtLjYgMC0xIC41LTEgMXYxLjc1YzAgLjUuNCAxIDEgMUgxM3YuNzVIOS41VjE1aDRjLjYgMCAxLS41IDEtMXYtMS43NWMwLS41LS40LTEtMS0xSDExdi0uNzVoMy41Vjl6bTUgMHY2SDE3di0yLjVoMlYxNWgxLjVWOUgxOXYyaC0yVjl6Ii8+PC9zdmc+", "properties": { "hostname": { "name": "hostname", "title": "Hostname", - "description": "Hostname to connect to. Usually in the form of an IP address", + "description": "Host name or IP address of the SSH server.", "type": "string", "parameterType": "string", "value": null, @@ -4592,7 +5679,7 @@ "port": { "name": "port", "title": "Port", - "description": "The port on which the connection will be tried on. Default is 22.", + "description": "TCP port the SSH server listens on.", "type": "string", "parameterType": "Long", "value": "22", @@ -4603,7 +5690,7 @@ "username": { "name": "username", "title": "Username", - "description": "The username with which a connection will be instantiated.", + "description": "Account to log in as.", "type": "string", "parameterType": "string", "value": null, @@ -4614,7 +5701,7 @@ "authentication_method": { "name": "authentication_method", "title": "Authentication method", - "description": "The method that is used to connect to the SSH server.", + "description": "How the task authenticates against the server.", "type": "string", "parameterType": "string", "value": "password", @@ -4625,7 +5712,7 @@ "private_key": { "name": "private_key", "title": "Private key", - "description": "Your private key to connect via SSH.", + "description": "Private key in PEM format, used when the authentication method is Key. RSA, ECDSA and Ed25519 keys are supported.", "type": "string", "parameterType": "password", "value": "", @@ -4636,7 +5723,7 @@ "password": { "name": "password", "title": "Password", - "description": "Depending on your authentication method this will either be used toconnect via password to SSH, or to decrypt the SSH private key", + "description": "Password of the account, or the passphrase of the private key when the authentication method is Key.", "type": "string", "parameterType": "password", "value": "", @@ -4647,7 +5734,7 @@ "path": { "name": "path", "title": "Path", - "description": "The currently selected path within your SSH instance. Auto-completion starts from user home folder, use '..' for parent directory or '/' for root directory.", + "description": "Remote directory the task works in. Autocompletion starts in the home directory of the account, use '..' for the parent directory or '/' for the root directory.", "type": "string", "parameterType": "string", "value": "", @@ -4657,8 +5744,8 @@ }, "error_handling": { "name": "error_handling", - "title": "Error handling for missing permissions.", - "description": "A choice on how to handle errors concerning the permissions rights.When choosing 'ignore' all files get listed regardless if the current user has correct permission rightsWhen choosing 'warning' all files get listed however there will be a mention that some of the files are not under the users permissionsif there are anyWhen choosing 'error' the files will not get listed if therethere are files the user has no access to.", + "title": "Error handling", + "description": "How the task reacts to files and folders the account cannot read.", "type": "string", "parameterType": "string", "value": "error", @@ -4669,7 +5756,7 @@ "no_subfolder": { "name": "no_subfolder", "title": "No subfolder", - "description": "When this flag is set, only files from the current directory will be listed.", + "description": "If enabled, only the given directory is listed and its subfolders are left out.", "type": "string", "parameterType": "boolean", "value": "false", @@ -4680,7 +5767,7 @@ "regex": { "name": "regex", "title": "Regular expression", - "description": "A regular expression used to define which files will get listed.", + "description": "Regular expression a file name has to match completely to be listed. It is matched against the plain file name, not against the path.", "type": "string", "parameterType": "string", "value": "^.*$", @@ -4692,8 +5779,8 @@ "properties_advanced": { "max_workers": { "name": "max_workers", - "title": "Maximum amount of workers.", - "description": "Determines the amount of workers used for concurrent thread execution of the task. Default is 1, maximum is 32. Note that too many workers can cause a ChannelException.", + "title": "Maximum number of workers", + "description": "Number of threads that list subfolders in parallel, from 1 to 32. More threads speed up a deep folder tree, but too many parallel channels make some servers refuse them with a ChannelException.", "type": "string", "parameterType": "Long", "value": "1", @@ -4705,7 +5792,7 @@ "actions": { "preview_results": { "label": "Preview results (max. 10)", - "description": "Lists 10 files as a preview.", + "description": "Connect to the server and list the first ten matching files.", "icon": null } }, @@ -5249,14 +6336,14 @@ "Uncategorized" ], "main_category": "Uncategorized", - "description": "Parses files, source code or input values as YAML documents.", - "markdownDocumentation": "\nThis workflow task parses YAML content from multiple sources and converts it to various output\nformats.\n\n**Input Sources:**\n\n- **entities**: Parse YAML from input port entities in a workflow\n- **code**: Parse YAML from directly entered source code\n- **file**: Parse YAML from uploaded project file resources\n\n**Output Formats:**\n\n- **entities**: Convert parsed structure to entities for workflow processing\n- **json_entities**: Output as single JSON entity to the output port\n- **json_dataset**: Save parsed structure directly to a JSON dataset\n\nThe plugin provides flexible YAML-to-JSON conversion with configurable input schema\ntypes and paths for entity-based processing. It includes comprehensive validation and\nerror handling for all supported modes.\n", + "description": "Parses YAML documents from a code field, entities or files and converts them to JSON.", + "markdownDocumentation": "\nThis task reads YAML documents, converts each of them to JSON, and hands the result on\neither as entities or as files.\n\nThe YAML comes from one of three places: the code field of this task, entities carrying it\nas text, or files arriving on an input port. Both port modes read every entity of every\nport the task declares, so a batch of files becomes a batch of results, and the number of\nthose ports is configurable, for documents which come from several tasks at once.\nConfigured to read from its code field, it declares no input port and starts the workflow.\nOn the output side the task either hands on the structure of the documents as entities, or\none JSON file per document.\n\nIt usually sits between a task that produces files and one that consumes them: read a\ndirectory of YAML, parse it here, and let a task that stores project resources keep the\nJSON. Feeding the entities output into a transformation is the other common chain.\n\nWorth knowing before configuring it:\n\n- A document has to describe a mapping or a sequence. A file holding nothing but a string\n or a number is rejected, since neither becomes a JSON object.\n- Exactly one document is read per file. A stream of several documents separated by `---`\n fails.\n- Parsing is safe, so YAML tags that construct arbitrary Python objects are refused.\n- Documents leaving as entities are combined into one stream whose paths are the union of\n all of them, and a value a document does not carry becomes empty. Where two documents\n disagree about a key - a string in one, a list in the next - the key is carried as text\n in both, since one schema cannot hold it both ways.\n- A key which is not a string becomes one. Mind that `on`, `yes` and `no` are booleans in\n YAML, so `on:` reads back as `True` unless it is quoted in the source document.\n- A returned file is named after the file it came from - after the entry, when that file\n is an entry in an archive - with a `.json` suffix, made unique when two of them would\n otherwise share a name. A document that did not come from a file - from the code field,\n or from an entity - has no name of its own and is written as `parsed-yaml.json` instead.\n", "pluginIcon": "data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiIHN0YW5kYWxvbmU9Im5vIj8+CjxzdmcKICAgd2lkdGg9IjkwLjg1ODYwNG1tIgogICBoZWlnaHQ9Ijk3LjIxMDhtbSIKICAgdmlld0JveD0iMCAwIDkwLjg1ODYwNSA5Ny4yMTA4MDEiCiAgIHZlcnNpb249IjEuMSIKICAgaWQ9ImNtZW0tcGx1Z2luLXlhbWwtbG9nbyIKICAgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KICA8cG9seWdvbgogICAgIHBvaW50cz0iMjU4LDEgMjU4LDE5IDMxOCwxMDkgMSwxMDkgNzMsMSAiCiAgICAgc3R5bGU9ImZpbGw6bm9uZSIKICAgICBpZD0icG9seWdvbjkxNyIKICAgICB0cmFuc2Zvcm09Im1hdHJpeCgwLjM0NjE5NDQzLDAsMCwwLjI5MDM0OTgxLC00LjMwODY0MTQsMjAuMjMwODgyKSIgLz4KICA8cG9seWdvbgogICAgIHBvaW50cz0iMSwxMDkgMzcsNTUgMSwxIDI1LDEgNDksMzcgNzMsMSA5NywxIDI1LDEwOSAiCiAgICAgaWQ9InBvbHlnb245MTkiCiAgICAgdHJhbnNmb3JtPSJtYXRyaXgoMC4zNDYxOTQ0MywwLDAsMC4yOTAzNDk4MSwxNy41NDA3MzQsMTYuOTIwMzY5KSIgLz4KICA8cG9seWdvbgogICAgIHBvaW50cz0iMTAzLDM3IDEwMywxMDkgMTI3LDEwOSAxMjcsMSAxMDMsMSAzMSwxMDkgNTUsMTA5ICIKICAgICBpZD0icG9seWdvbjkyMSIKICAgICB0cmFuc2Zvcm09Im1hdHJpeCgwLjM0NjE5NDQzLDAsMCwwLjI5MDM0OTgxLDE3LjU0MDczNCwxNi45MjAzNjkpIiAvPgogIDxwb2x5Z29uCiAgICAgcG9pbnRzPSIxNTYsMSAxMzIsMSAxMzIsMTA5IDE1NiwxMDkgMTU2LDM3IDE4MCw3MyAyMDQsMzcgMjA0LDEwOSAyMjgsMTA5IDIyOCwxIDE5OCwxIDE4MCwzNyAiCiAgICAgaWQ9InBvbHlnb245MjMiCiAgICAgdHJhbnNmb3JtPSJtYXRyaXgoMC4zNDYxOTQ0MywwLDAsMC4yOTAzNDk4MSwtMjguMTQ0MzMsNTAuMDI1NDk4KSIgLz4KICA8cG9seWdvbgogICAgIHBvaW50cz0iMjU4LDEgMjM0LDEgMjM0LDEwOSAzMTgsMTA5IDMwMiw4NSAyNTgsODUgIgogICAgIGlkPSJwb2x5Z29uOTI1IgogICAgIHRyYW5zZm9ybT0ibWF0cml4KDAuMzQ2MTk0NDMsMCwwLDAuMjkwMzQ5ODEsLTI4LjE0NDMzLDUwLjAyNTQ5OCkiIC8+Cjwvc3ZnPgo=", "properties": { "source_mode": { "name": "source_mode", "title": "Source / Input Mode", - "description": "", + "description": "Where the YAML documents come from.", "type": "string", "parameterType": "string", "value": "code", @@ -5267,7 +6354,7 @@ "target_mode": { "name": "target_mode", "title": "Target / Output Mode", - "description": "", + "description": "In which shape the parsed documents leave this task.", "type": "string", "parameterType": "string", "value": "entities", @@ -5278,7 +6365,7 @@ "source_code": { "name": "source_code", "title": "YAML Source Code (when using the *code* input)", - "description": "", + "description": "The YAML document itself, used in the 'code' source mode.", "type": "string", "parameterType": "code-yaml", "value": "# Add your YAML code here (and select 'code' as input mode).", @@ -5288,24 +6375,24 @@ } }, "properties_advanced": { - "source_file": { - "name": "source_file", - "title": "YAML File (when using the *file* input)", - "description": "Which YAML file do you want to load into a JSON dataset? The dropdown shows file resources from the current project.", + "number_of_inputs": { + "name": "number_of_inputs", + "title": "Number of Input Ports", + "description": "How many input ports the task declares. Raise it to read from several tasks at once, in the order the ports are shown. It has no effect in the 'code' source mode, which declares no input port at all.", "type": "string", - "parameterType": "string", - "value": "", + "parameterType": "Long", + "value": "1", "advanced": true, "visibleInDialog": true, "properties": {} }, - "target_dataset": { - "name": "target_dataset", - "title": "Target Dataset", - "description": "Where do you want to save the result of the conversion? The dropdown shows JSON datasets from the current project.", + "tolerate_unusable_input": { + "name": "tolerate_unusable_input", + "title": "Tolerate Unusable Input", + "description": "If enabled, a document that cannot be parsed is reported and skipped instead of stopping the task, and an input that delivers nothing at all produces an empty result. A batch in which every document fails remains an error, since something arrived and none of it could be used.", "type": "string", - "parameterType": "string", - "value": "", + "parameterType": "boolean", + "value": "false", "advanced": true, "visibleInDialog": true, "properties": {} @@ -5313,7 +6400,7 @@ "input_schema_type": { "name": "input_schema_type", "title": "Input Schema Type / Class", - "description": "In case of source mode 'entities', you can specify the requested input type.", + "description": "The type the input ports ask for, used in the 'entities' source mode. Change it when the preceding task delivers entities of a different type.", "type": "string", "parameterType": "string", "value": "urn:x-eccenca:yaml-document", @@ -5324,7 +6411,7 @@ "input_schema_path": { "name": "input_schema_path", "title": "Input Schema Path / Property", - "description": "In case of source mode 'entities', you can specify the requested input path.", + "description": "The path the YAML document is read from, used in the 'entities' source mode. Change it when the preceding task carries the document in a differently named path.", "type": "string", "parameterType": "string", "value": "text", @@ -5505,16 +6592,16 @@ ], "main_category": "Uncategorized", "description": "Performs OWL reasoning.", - "markdownDocumentation": "A task performing OWL reasoning. With an OWL ontology and a data graph as input the reasoning result is written to a specified graph.\n \n## Options\n\n### Data graph IRI\n\nThe IRI of the input data graph. The graph IRI is selected from a list of graphs of types `di:Dataset`, `void:Dataset`\nand `owl:Ontology`.\n\n### Ontology graph IRI\n\nThe IRI of the input ontology graph. The graph IRI is selected from a list of graphs of type`owl:Ontology`.\n\n### Ignore missing imports\n\nIf enabled, missing imports (`owl:imports`) in the input graphs are ignored.\n\n### Output graph IRI\n\nThe IRI of the output graph for the reasoning result.\n\n\u26a0\ufe0f Existing graphs will be overwritten.\n\n### Reasoner\n\nThe following reasoner options are supported: \n- [ELK](https://code.google.com/p/elk-reasoner/) (elk)\n- [Expression Materializing Reasoner](http://static.javadoc.io/org.geneontology/expression-materializing-reasoner/0.1.3/org/geneontology/reasoner/ExpressionMaterializingReasoner.html) (emr)\n- [HermiT](http://www.hermit-reasoner.com/) (hermit)\n- [JFact](http://jfact.sourceforge.net/) (jfact)\n- [Structural Reasoner](http://owlcs.github.io/owlapi/apidocs_4/org/semanticweb/owlapi/reasoner/structural/StructuralReasoner.html) (structural)\n- [Whelk](https://github.com/balhoff/whelk) (whelk)\n\n### Generated Axioms\n\nThe plugin provides the following parameters to include inferred axiom generators:\n\n#### Class axiom generators\n- **Class inclusion (rdfs:subClassOf)** \nThe reasoner will infer assertions about the hierarchy of classes, i.e.\n`SubClassOf:` statements. \nIf there are classes `Person`, `Student` and `Professor`, such that `Person DisjointUnionOf:\nStudent, Professor` holds, the reasoner will infer `Student SubClassOf: Person`. \n\n\n- **Class equivalence (owl:equivalentClass)** \nThe reasoner will infer assertions about the equivalence of classes, i.e.\n`EquivalentTo:` statements. \nIf there are classes `Person`, `Student` and `Professor`, such that `Person DisjointUnionOf:\nStudent, Professor` holds, the reasoner will infer `Person EquivalentTo: Student and Professor`.\n\n\n- **Class disjointness (owl:disjointWith)** \nThe reasoner will infer assertions about the disjointness of classes, i.e.\n`DisjointClasses:` statements. \nIf there are classes `Person`, `Student` and `Professor`, such that `Person DisjointUnionOf:\nStudent, Professor` holds, the reasoner will infer `DisjointClasses: Student, Professor`.\n\n \n- **Data property equivalence (owl:equivalentProperty)** \nThe reasoner will infer axioms about the equivalence of data properties,\n i.e. `EquivalentProperties` statements. \nIf there are data properties `identifier` and `enrollmentNumber`, such that `enrollmentNumber\nSubPropertyOf: identifier` and `identifier SubPropertyOf: enrollmentNumber` holds, the reasoner\nwill infer `Student EquivalentProperties: identifier, enrollmentNumber`.\n\n\n- **Data property inclusion (rdfs:subPropertyOf)** \nThe reasoner will infer axioms about the hierarchy of data properties,\ni.e. `SubPropertyOf:` statements. \nIf there are data properties `identifier`, `studentIdentifier` and `enrollmentNumber`, such that\n`studentIdentifier SubPropertyOf: identifier` and `enrollmentNumber SubPropertyOf:\nstudentIdentifier` holds, the reasoner will infer `enrollmentNumber SubPropertyOf: identifier`.\n\n\n#### Individual axiom generators\n- **Individual class assertions (rdf:type)** \nThe reasoner will infer assertions about the classes of individuals, i.e.\n`Types:` statements. \nAssume, there are classes `Person`, `Student` and `University` as well as the property\n`enrolledIn`, such that `Student EquivalentTo: Person and enrolledIn some University` holds. For\nthe individual `John` with the assertions `John Types: Person; Facts: enrolledIn\nLeipzigUniversity`, the reasoner will infer `John Types: Student`.\n\n\n- **Individual property assertions** \nThe reasoner will infer assertions about the properties of individuals,\ni.e. `Facts:` statements. \nAssume, there are properties `enrolledIn` and `offers`, such that `enrolled SubPropertyChain:\nenrolledIn o inverse (offers)` holds. For the individuals `John`and `LeipzigUniversity` with the\nassertions `John Facts: enrolledIn KnowledgeRepresentation` and `LeipzigUniversity Facts: offers\nKnowledgeRepresentation`, the reasoner will infer `John Facts: enrolledIn LeipzigUniversity`.\n\n\n#### Object property axiom generators\n- **Object property equivalence (owl:equivalentProperty)** \nThe reasoner will infer assertions about the equivalence of object\nproperties, i.e. `EquivalentTo:` statements. \nIf there are object properties `hasAlternativeLecture` and `hasSameTopicAs`, such that\n`hasAlternativeLecture Characteristics: Symmetric` and `hasSameTopicAs InverseOf:\nhasAlternativeLecture` holds, the reasoner will infer `EquivalentProperties: hasAlternativeLecture,\nhasSameTopicAs`.\n\n\n- **Object property inversion (owl:inverseOf)** \nThe reasoner will infer axioms about the inversion about object\nproperties, i.e. `InverseOf:` statements. \nIf there is a object property `hasAlternativeLecture`, such that `hasAlternativeLecture\nCharacteristics: Symmetric` holds, the reasoner will infer `hasAlternativeLecture InverseOf:\nhasAlternativeLecture`.\n\n\n- **Object property inclusion (rdfs:subPropertyOf)** \nThe reasoner will infer axioms about the inclusion of object properties,\ni.e. `SubPropertyOf:` statements. \nIf there are object properties `enrolledIn`, `studentOf` and `hasStudent`, such that `enrolledIn\nSubPropertyOf: studentOf` and `enrolledIn InverseOf: hasStudent` holds, the reasoner will infer\n`hasStudent SubPropertyOf: inverse (studentOf)`.\n\n\n- **Object property ranges (rdfs:range)** \nThe reasoner will infer axioms about the ranges of object properties,\ni.e. `Range:` statements. \nIf there are classes `Student` and `Lecture` as wells as object properties `hasStudent` and\n`enrolledIn`, such that `hasStudent Range: Student and enrolledIn some Lecture` holds, the\nreasoner will infer `hasStudent Range: Student`.\n\n\n- **Object property domains (rdfs:domain)** \nThe reasoner will infer axioms about the domains of object\nproperties, i.e. `Domain:` statements. \nIf there are classes `Person`, `Student` and `Professor` as wells as the object property\n`hasRoleIn`, such that `Professor SubClassOf: Person`, `Student SubClassOf: Person` and\n`hasRoleIn Domain: Professor or Student` holds, the reasoner will infer `hasRoleIn Domain: Person`.\n\n\n### Validate OWL2 profiles\n\nValidate the input ontology against OWL profiles (DL, EL, QL, RL, and Full) and annotate the result graph. \n\n### Process valid OWL profiles from input\n\nIf enabled along with the \"Validate OWL2 profiles\" parameter, the valid profiles, ontology IRI and reasoner option is\ntaken from the config port input (parameters \"valid_profiles\", \"ontology_graph_iri\" and \"reasoner\") and the OWL2\nprofiles validation is not done in the plugin. The valid profiles input is a comma-separated string (e.g. \"Full,DL\").\n\n### Output graph import\n\nAdd the triple owl:imports into the output graph or add the triple\n owl:imports into the ontology graph.\n\n### Maximum RAM Percentage\n\nMaximum heap size for the Java virtual machine in the DI container running the reasoning process.\n\n\u26a0\ufe0f Setting the percentage too high may result in an out of memory error.\n", + "markdownDocumentation": "A task performing OWL reasoning. With an OWL ontology and/or a data graph as input the reasoning result is written to a\nspecified graph. At least one of the two input graphs must be specified. With two different input graphs, the data\ngraph is reasoned with the ontology imported; with only one, or the same graph in both, that graph is reasoned on its\nown.\n \n## Options\n\n### Data graph IRI\n\nThe IRI of the input data graph. The graph IRI is selected from a list of graphs of types `di:Dataset`, `void:Dataset`\nand `owl:Ontology`. Optional if an ontology graph is specified.\n\n### Ontology graph IRI\n\nThe IRI of the input ontology graph. The graph IRI is selected from a list of graphs of type `owl:Ontology`. Optional if\na data graph is specified.\n\n### Ignore missing imports\n\nIf enabled, missing imports (`owl:imports`) in the input graphs are ignored.\n\n### Output graph IRI\n\nThe IRI of the output graph for the reasoning result.\n\n\u26a0\ufe0f Existing graphs will be overwritten.\n\n### Reasoner\n\nThe following reasoner options are supported: \n- [ELK](https://code.google.com/p/elk-reasoner/) (elk)\n- [Expression Materializing Reasoner](http://static.javadoc.io/org.geneontology/expression-materializing-reasoner/0.1.3/org/geneontology/reasoner/ExpressionMaterializingReasoner.html) (emr)\n- [HermiT](http://www.hermit-reasoner.com/) (hermit)\n- [JFact](http://jfact.sourceforge.net/) (jfact)\n- [Structural Reasoner](http://owlcs.github.io/owlapi/apidocs_4/org/semanticweb/owlapi/reasoner/structural/StructuralReasoner.html) (structural)\n- [Whelk](https://github.com/balhoff/whelk) (whelk)\n\n### Generated Axioms\n\nThe plugin provides the following parameters to include inferred axiom generators:\n\n#### Class axiom generators\n- **Class inclusion (rdfs:subClassOf)** \nThe reasoner will infer assertions about the hierarchy of classes, i.e.\n`SubClassOf:` statements. \nIf there are classes `Person`, `Student` and `Professor`, such that `Person DisjointUnionOf:\nStudent, Professor` holds, the reasoner will infer `Student SubClassOf: Person`. \n\n\n- **Class equivalence (owl:equivalentClass)** \nThe reasoner will infer assertions about the equivalence of classes, i.e.\n`EquivalentTo:` statements. \nIf there are classes `Person`, `Student` and `Professor`, such that `Person DisjointUnionOf:\nStudent, Professor` holds, the reasoner will infer `Person EquivalentTo: Student and Professor`.\n\n\n- **Class disjointness (owl:disjointWith)** \nThe reasoner will infer assertions about the disjointness of classes, i.e.\n`DisjointClasses:` statements. \nIf there are classes `Person`, `Student` and `Professor`, such that `Person DisjointUnionOf:\nStudent, Professor` holds, the reasoner will infer `DisjointClasses: Student, Professor`.\n\n \n- **Data property equivalence (owl:equivalentProperty)** \nThe reasoner will infer axioms about the equivalence of data properties,\n i.e. `EquivalentProperties` statements. \nIf there are data properties `identifier` and `enrollmentNumber`, such that `enrollmentNumber\nSubPropertyOf: identifier` and `identifier SubPropertyOf: enrollmentNumber` holds, the reasoner\nwill infer `Student EquivalentProperties: identifier, enrollmentNumber`.\n\n\n- **Data property inclusion (rdfs:subPropertyOf)** \nThe reasoner will infer axioms about the hierarchy of data properties,\ni.e. `SubPropertyOf:` statements. \nIf there are data properties `identifier`, `studentIdentifier` and `enrollmentNumber`, such that\n`studentIdentifier SubPropertyOf: identifier` and `enrollmentNumber SubPropertyOf:\nstudentIdentifier` holds, the reasoner will infer `enrollmentNumber SubPropertyOf: identifier`.\n\n\n#### Individual axiom generators\n- **Individual class assertions (rdf:type)** \nThe reasoner will infer assertions about the classes of individuals, i.e.\n`Types:` statements. \nAssume, there are classes `Person`, `Student` and `University` as well as the property\n`enrolledIn`, such that `Student EquivalentTo: Person and enrolledIn some University` holds. For\nthe individual `John` with the assertions `John Types: Person; Facts: enrolledIn\nLeipzigUniversity`, the reasoner will infer `John Types: Student`.\n\n\n- **Individual property assertions** \nThe reasoner will infer assertions about the properties of individuals,\ni.e. `Facts:` statements. \nAssume, there are properties `enrolledIn` and `offers`, such that `enrolled SubPropertyChain:\nenrolledIn o inverse (offers)` holds. For the individuals `John`and `LeipzigUniversity` with the\nassertions `John Facts: enrolledIn KnowledgeRepresentation` and `LeipzigUniversity Facts: offers\nKnowledgeRepresentation`, the reasoner will infer `John Facts: enrolledIn LeipzigUniversity`.\n\n\n#### Object property axiom generators\n- **Object property equivalence (owl:equivalentProperty)** \nThe reasoner will infer assertions about the equivalence of object\nproperties, i.e. `EquivalentTo:` statements. \nIf there are object properties `hasAlternativeLecture` and `hasSameTopicAs`, such that\n`hasAlternativeLecture Characteristics: Symmetric` and `hasSameTopicAs InverseOf:\nhasAlternativeLecture` holds, the reasoner will infer `EquivalentProperties: hasAlternativeLecture,\nhasSameTopicAs`.\n\n\n- **Object property inversion (owl:inverseOf)** \nThe reasoner will infer axioms about the inversion about object\nproperties, i.e. `InverseOf:` statements. \nIf there is a object property `hasAlternativeLecture`, such that `hasAlternativeLecture\nCharacteristics: Symmetric` holds, the reasoner will infer `hasAlternativeLecture InverseOf:\nhasAlternativeLecture`.\n\n\n- **Object property inclusion (rdfs:subPropertyOf)** \nThe reasoner will infer axioms about the inclusion of object properties,\ni.e. `SubPropertyOf:` statements. \nIf there are object properties `enrolledIn`, `studentOf` and `hasStudent`, such that `enrolledIn\nSubPropertyOf: studentOf` and `enrolledIn InverseOf: hasStudent` holds, the reasoner will infer\n`hasStudent SubPropertyOf: inverse (studentOf)`.\n\n\n- **Object property ranges (rdfs:range)** \nThe reasoner will infer axioms about the ranges of object properties,\ni.e. `Range:` statements. \nIf there are classes `Student` and `Lecture` as wells as object properties `hasStudent` and\n`enrolledIn`, such that `hasStudent Range: Student and enrolledIn some Lecture` holds, the\nreasoner will infer `hasStudent Range: Student`.\n\n\n- **Object property domains (rdfs:domain)** \nThe reasoner will infer axioms about the domains of object\nproperties, i.e. `Domain:` statements. \nIf there are classes `Person`, `Student` and `Professor` as wells as the object property\n`hasRoleIn`, such that `Professor SubClassOf: Person`, `Student SubClassOf: Person` and\n`hasRoleIn Domain: Professor or Student` holds, the reasoner will infer `hasRoleIn Domain: Person`.\n\n\n### Validate OWL2 profiles\n\nValidate the input ontology against OWL profiles (DL, EL, QL, RL, and Full) and annotate the result graph.\nNeeds an ontology graph.\n\n### Process valid OWL profiles from input\n\nIf enabled along with the \"Validate OWL2 profiles\" parameter, the valid profiles, ontology IRI and reasoner option is\ntaken from the config port input (parameters \"valid_profiles\", \"ontology_graph_iri\" and \"reasoner\") and the OWL2\nprofiles validation is not done in the plugin. The valid profiles input is a comma-separated string (e.g. \"Full,DL\").\n\n### Output graph import\n\nAdd the triple owl:imports into the output graph or add the triple\n owl:imports into the ontology graph. Needs an ontology graph.\n\n### Maximum RAM Percentage\n\nMaximum heap size for the Java virtual machine in the DI container running the reasoning process.\n\n\u26a0\ufe0f Setting the percentage too high may result in an out of memory error.\n", "pluginIcon": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIxOTIiIGhlaWdodD0iMTkyIiB2aWV3Qm94PSItMiAwIDI4IDI0Ij4KCTxwYXRoIGZpbGw9ImN1cnJlbnRDb2xvciIgZD0iTTcuMTUyIDMuMDEyYy44MzItLjY0OCAxLjkyLTEuMDA2IDMuMDAxLTEuMDA2Yy42NDcgMCAxLjE3LjIyMiAxLjU2Ny41NzVxLjE2LjE0NC4yODcuMzA0cS4xMjctLjE2LjI4Ny0uMzA0Yy4zOTctLjM1My45Mi0uNTc1IDEuNTY3LS41NzVjMS4wODEgMCAyLjE3LjM1OCAzIDEuMDA2Yy42NDguNTAzIDEuMTU1IDEuMTk4IDEuMzU0IDIuMDM3Yy40Mi4wNy43OTMuMjg0IDEuMDk1LjU2N2MuNDguNDUxLjgyMyAxLjEwMyAxLjAzOCAxLjc3NGMuMjE5LjY4MS4zMjggMS40NDYuMjk1IDIuMTgxYy0uMDE3LjM3Ni0uMDcyLjc2LS4xNzcgMS4xMjdsLjA2NS4wM2MuMzcuMTc0LjY3LjQ0Ny44OTUuODFjLjQyNS42ODUuNTc1IDEuNjcxLjU3NSAyLjkyOGMwIDEuNDQ1LS41NTMgMi40MjYtMS4yNjMgMy4wMzVhMy40NSAzLjQ1IDAgMCAxLTEuMjcuNjlhNS4yIDUuMiAwIDAgMS0xLjAxOCAyLjEzN2MtLjcyMy45MDQtMS44NDcgMS42NjgtMy4zNTcgMS42NjhjLTEuMjEgMC0yLjE2My0uNjctMi43NzUtMS4zMWE1IDUgMCAwIDEtLjMxLS4zNTdhNSA1IDAgMCAxLS4zMTEuMzU2Yy0uNjEyLjY0Mi0xLjU2NSAxLjMxMS0yLjc3NSAxLjMxMWMtMS41MTEgMC0yLjYzNS0uNzY0LTMuMzU3LTEuNjY4YTUuMiA1LjIgMCAwIDEtMS4wMi0yLjEzN2EzLjQ1IDMuNDUgMCAwIDEtMS4yNjktLjY5Yy0uNzEtLjYxLTEuMjYyLTEuNTktMS4yNjItMy4wMzVjMC0xLjI1Ny4xNS0yLjI0My41NzQtMi45MjhhMi4xIDIuMSAwIDAgMSAuOTYtLjg0YTUgNSAwIDAgMS0uMTc3LTEuMTI3Yy0uMDMzLS43MzUuMDc2LTEuNS4yOTUtMi4xODFjLjIxNS0uNjcuNTU3LTEuMzIzIDEuMDM4LTEuNzc0YTIuMSAyLjEgMCAwIDEgMS4wOTUtLjU2N2MuMi0uODQuNzA2LTEuNTM0IDEuMzUzLTIuMDM3bS45MjIgMS4xODNjLS41NDYuNDI1LS44NjYuOTkxLS44NjYgMS42NDNhLjc1Ljc1IDAgMCAxLS45ODcuNzExYy0uMTM3LS4wNDYtLjI4Mi0uMDM1LS40OS4xNjFjLS4yMzIuMjE3LS40NjcuNjEtLjYzNyAxLjEzOGE0LjggNC44IDAgMCAwLS4yMjQgMS42NTZjLjAyNS41NTYuMTU3IDEuMDE4LjM2NSAxLjMzYS44LjggMCAwIDEgLjA4My4xNjZINi40YTIuODUgMi44NSAwIDAgMSAyLjg0MyAyLjY0MmEyIDIgMCAxIDEtMS41MDcuMDFBMS4zNSAxLjM1IDAgMCAwIDYuNCAxMi41SDMuNzc1Yy0uMTUuMzQ4LS4yNjEuOTQ5LS4yNjEgMS45NjZjMCAxLjAyMS4zNzEgMS41ODEuNzM5IDEuODk3Yy40MDMuMzQ1Ljg0Ni40MzUuOTc0LjQzNWEuNzUuNzUgMCAwIDEgLjc1Ljc1YzAgLjQyLjIzMiAxLjE4My43NiAxLjg0M2MuNTA4LjYzNyAxLjIzMiAxLjEwNSAyLjE4NSAxLjEwNWMuNjM3IDAgMS4yMjMtLjM1OCAxLjY5LS44NDZjLjIyNi0uMjM4LjQwMi0uNDg1LjUxOC0uNjg1YTIgMiAwIDAgMCAuMTE1LS4yMzNsLjAwNi0uMDE0VjkuMjVoLS44OTZhMiAyIDAgMSAxIDAtMS41aC44OTZWNS4yMTJsLS4wMDMtLjA1N2EzLjMgMy4zIDAgMCAwLS4xOC0uOTAzYy0uMDktLjI0LS4yMDctLjQyNy0uMzQ1LS41NWMtLjEyMy0uMTEtLjI5My0uMTk2LS41Ny0uMTk2Yy0uNzY2IDAtMS41MjUuMjU4LTIuMDguNjlNMTIuNzY0IDE3djEuNzE4bC4wMDUuMDE0cS4wMy4wOC4xMTYuMjMzYy4xMTUuMi4yOTEuNDQ3LjUxOC42ODVjLjQ2Ni40ODggMS4wNTMuODQ2IDEuNjkuODQ2Yy45NTIgMCAxLjY3Ni0uNDY4IDIuMTg1LTEuMTA1Yy41MjctLjY2Ljc2LTEuNDI0Ljc2LTEuODQzYS43NS43NSAwIDAgMSAuNzUtLjc1Yy4xMjggMCAuNTctLjA5Ljk3NC0uNDM1Yy4zNjctLjMxNi43MzgtLjg3Ni43MzgtMS44OTdjMC0xLjIwOC0uMTU4LTEuODMtLjM0OS0yLjEzN2EuNi42IDAgMCAwLS4yNi0uMjQ1Yy0uMTAxLS4wNDgtLjI1My0uMDg0LS40ODgtLjA4NGEuNzUuNzUgMCAwIDEtLjYyNC0xLjE2NmMuMjA4LS4zMTIuMzQtLjc3NC4zNjYtMS4zM2E0LjggNC44IDAgMCAwLS4yMjUtMS42NTZjLS4xNy0uNTI4LS40MDUtLjkyLS42MzYtMS4xMzhjLS4yMDktLjE5Ni0uMzU0LS4yMDctLjQ5LS4xNjFhLjc1Ljc1IDAgMCAxLS45ODgtLjcxMWMwLS42NTItLjMyLTEuMjE4LS44NjYtMS42NDNjLS41NTMtLjQzMS0xLjMxMy0uNjktMi4wOC0uNjljLS4yNzYgMC0uNDQ1LjA4Ny0uNTY5LjE5NmExLjQgMS40IDAgMCAwLS4zNDQuNTVhMyAzIDAgMCAwLS4xODMuOTYxVjE1LjVoLjYzNmExLjM1IDEuMzUgMCAwIDAgMS4zNS0xLjM1di0xLjc5NWEyIDIgMCAxIDEgMS41IDB2MS43OTVBMi44NSAyLjg1IDAgMCAxIDEzLjQgMTd6TTguNSA4YS41LjUgMCAxIDAgMCAxYS41LjUgMCAwIDAgMC0xTTggMTUuNWEuNS41IDAgMSAwIDEgMGEuNS41IDAgMCAwLTEgMG03LTVhLjUuNSAwIDEgMCAxIDBhLjUuNSAwIDAgMC0xIDAiIC8+Cjwvc3ZnPg==", "properties": { "data_graph_iri": { "name": "data_graph_iri", "title": "Data graph IRI", - "description": "The IRI of the input data graph.", + "description": "The IRI of the input data graph. At least one of data graph and ontology graph must be specified.", "type": "string", "parameterType": "scheme:string", - "value": null, + "value": "", "advanced": false, "visibleInDialog": true, "properties": {} @@ -5522,10 +6609,10 @@ "ontology_graph_iri": { "name": "ontology_graph_iri", "title": "Ontology graph IRI", - "description": "The IRI of the input ontology graph.", + "description": "The IRI of the input ontology graph. At least one of data graph and ontology graph must be specified.", "type": "string", "parameterType": "scheme:string", - "value": null, + "value": "", "advanced": false, "visibleInDialog": true, "properties": {} @@ -5709,7 +6796,7 @@ "imports": { "name": "imports", "title": "Output graph import", - "description": "Add the triple owl:imports to the output graph or add the triple owl:imports to the ontology graph.", + "description": "Add the triple owl:imports to the output graph or add the triple owl:imports to the ontology graph. Needs an ontology graph.", "type": "string", "parameterType": "string", "value": "none", @@ -5755,8 +6842,6 @@ }, "actions": {}, "required": [ - "data_graph_iri", - "ontology_graph_iri", "output_graph_iri", "reasoner" ], @@ -5801,7 +6886,7 @@ ], "main_category": "Uncategorized", "description": "Run an Apify actor and deliver its dataset as a file.", - "markdownDocumentation": "\nThis task drives [Apify](https://apify.com) from a workflow. It starts an actor\nrun, waits for it, and hands the dataset the run produced on as a file.\n\nThe actor input is a JSON document written as a Jinja template. Whatever the\ntemplate refers to becomes the input port of this task, so the values come from\nthe entities of the preceding operator - the knowledge graph decides what is\nscraped. A template that refers to nothing has no input port and is sent as it\nstands. Substituted values need the `tojson` filter, as in\n`\"query\": {{ companyName | tojson }}`, which quotes and escapes them correctly\nand renders a list where a list is supplied.\n\nThe result leaves through the file port as one file per actor run, in the chosen\nformat. It is deliberately not turned into entities here: a dataset can be very\nlarge, and reading structure out of a file is what the JSON, CSV and XML\ndatasets of Corporate Memory already do.\n\nThe task covers three cases, which change its shape. Reading an existing Apify\ndataset starts no run and needs no input. Running an actor without reading it\ndelivers a record of the run instead of a file. Running and reading does both in\none step.\n\nCaveats worth knowing:\n\n* Editing the actor input changes the input port, which can invalidate the\n mapping of a workflow that was already saved.\n* Values are read from the input entities by path name, so a path has to be a\n name Jinja accepts. Rename it in a preceding operator if it is a URI.\n* With one run per entity, a path holding several values contributes only its\n first; the report says how often that happened.\n* A build tag such as `latest` satisfies the Build parameter but pins nothing -\n it follows whatever the actor's author publishes next. A version number pins.\n* Downloaded files stay in the temporary directory of the worker.\n ", + "markdownDocumentation": "\nThis task drives [Apify](https://apify.com) from a workflow. It starts an actor\nrun, waits for it, and hands the dataset the run produced on as a file.\n\nThe actor input is a JSON document written as a Jinja template. Whatever the\ntemplate refers to becomes the input port of this task, so the values come from\nthe entities of the preceding operator - the knowledge graph decides what is\nscraped. A template that refers to nothing has no input port and is sent as it\nstands. Substituted values need the `tojson` filter, as in\n`\"query\": {{ companyName | tojson }}`, which quotes and escapes them correctly\nand renders a list where a list is supplied.\n\nThe result leaves through the file port as one file per actor run, in the chosen\nformat. It is deliberately not turned into entities here: a dataset can be very\nlarge, and reading structure out of a file is what the JSON, CSV and XML\ndatasets of eccenca Corporate Memory already do.\n\nThe task covers three cases, which change its shape. Reading an existing Apify\ndataset starts no run and needs no input. Running an actor without reading it\ndelivers a record of the run instead of a file. Running and reading does both in\none step.\n\nCaveats worth knowing:\n\n* Editing the actor input changes the input port, which can invalidate the\n mapping of a workflow that was already saved.\n* Values are read from the input entities by path name, so a path has to be a\n name Jinja accepts. Rename it in a preceding operator if it is a URI.\n* With one run per entity, a path holding several values contributes only its\n first; the report says how often that happened.\n* A build tag such as `latest` satisfies the Build parameter but pins nothing -\n it follows whatever the actor's author publishes next. A version number pins.\n* Downloaded files stay in the temporary directory of the worker.\n ", "pluginIcon": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA2NCA2NCIgZmlsbD0iY3VycmVudENvbG9yIiByb2xlPSJpbWciIGFyaWEtbGFiZWw9IkFwaWZ5Ij4KICA8cGF0aCBkPSJNMzIgMTIuNSA0Ny44IDUxaC04LjFMMzIgMzEuNyAyNC4zIDUxaC04LjF6Ii8+CiAgPHBhdGggZD0iTTI0LjYgNDAuMmgxNC44bDIuNiA2LjRIMjJ6IiBvcGFjaXR5PSIuNTUiLz4KPC9zdmc+Cg==", "properties": { "api_token": { @@ -5936,7 +7021,7 @@ }, "preview_dataset": { "label": "Preview Apify dataset (max. 10)", - "description": "Show the first items of the selected Apify dataset.", + "description": "Show the first items of the selected Apify dataset. Available in Read mode only.", "icon": null } }, @@ -6171,57 +7256,237 @@ "visibleInDialog": true, "properties": {} }, - "limit": { - "name": "limit", - "title": "Limit", - "description": "Maximum number of log entries to retrieve. Must be a positive integer.", + "limit": { + "name": "limit", + "title": "Limit", + "description": "Maximum number of log entries to retrieve. Must be a positive integer.", + "type": "string", + "parameterType": "Long", + "value": "1000", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "repos": { + "name": "repos", + "title": "Repositories", + "description": "Comma-separated list of repository names to search. Leave no trailing comma here. Example: 'windows,linux,firewall'. Leave empty to search all accessible repositories. Use the 'Preview repositories' action to see available options.", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "paths_list": { + "name": "paths_list", + "title": "List of output paths", + "description": "Comma-separated list of field paths to include in output. Example: 'source_address, destination_address, user, log_ts'. Leave no trailing comma here. If specified, creates a fixed output schema with only these fields. Leave empty for automatic schema detection with all available fields. Use 'Preview output paths' action to discover available field names. See Logpoint field naming conventions at: https://docs.logpoint.com/docs/logpoint-taxonomy-guideline/en/latest/Field%20naming%20convention.html", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + } + }, + "properties_advanced": {}, + "actions": { + "preview_output_paths": { + "label": "Preview output paths", + "description": "Executes the configured query with a limit of 1 to retrieve and display available field paths in the result set. Use this to discover which fields you can specify in the 'List of output paths' parameter for schema customization.", + "icon": null + }, + "preview_repositories": { + "label": "Preview repositories", + "description": "Retrieves and displays all log repositories accessible with the configured credentials. Use this to identify repository names for the 'Repositories' parameter to filter searches to specific log sources.", + "icon": null + } + }, + "required": [ + "secret_key", + "query" + ], + "distanceMeasureRange": null, + "backendType": "python", + "is_deprecated": false, + "tags": [ + "WorkflowTask", + "PythonPlugin" + ], + "pluginType": "customtask", + "relatedPlugins": [] + }, + "cmem_plugin_ldap-Search": { + "pluginId": "cmem_plugin_ldap-Search", + "title": "Search LDAP directory", + "categories": [ + "Uncategorized" + ], + "main_category": "Uncategorized", + "description": "Query LDAP directory services such as OpenLDAP or Active Directory.", + "markdownDocumentation": "\nSearches a directory server over LDAP and returns each matching entry as an entity. It works with\nany LDAPv3 directory, among them OpenLDAP, Active Directory and Apple Open Directory.\n\n## What the task produces\n\nThe task consumes nothing and produces one entity per directory entry. Each entity carries the\ndistinguished name of the entry as its first value, followed by the values of the requested\nattributes, in the order they were requested.\n\nA search is normally followed by a transformation which maps the entries into a Knowledge Graph,\nwith the distinguished name serving as the stable identifier of a person, group or device.\n\n## How an entry is identified\n\nThe URI of an entity is the LDAP URL of the entry as defined by RFC 4516, built from the URL of\nthe directory and the percent-encoded distinguished name:\n\n```\nldaps://directory.example.org/uid%3Djdoe%2Cou%3Dpeople%2Cdc%3Dexample%2Cdc%3Dorg\n```\n\nSuch a URI stays the same between runs and identifies the entry on its own directory, so two\ndirectories holding the same distinguished name do not collide. The distinguished name is also\navailable as an ordinary value, which is what a mapping usually builds its own identifier from.\n\n## How values are returned\n\nAn LDAP attribute may hold several values, and all of them are returned. An attribute an entry\ndoes not carry produces no value at all, which keeps it distinguishable from an attribute that is\npresent and empty.\n\nValues arrive in the lexical form a Knowledge Graph expects: timestamps as ISO 8601 dates,\nbooleans as `true` and `false`, numbers in decimal. A value which is not text - `jpegPhoto`,\n`objectSid`, `objectGUID` or `userCertificate`, for instance - is Base64 encoded.\n\nThis works because the schema of the directory is read while connecting, at the cost of one\nrequest. A directory sends every value as text, and only its schema says which of them is a\ntimestamp, a boolean or a number.\n\n## Naming the attributes\n\nThe attributes have to be named one by one. `*` and `+` are rejected, because the entities are\ndescribed to the workflow while it is being drawn, and a wildcard is only resolved by the\ndirectory once the search runs - there would be nothing to connect the next task to.\n\nDirectories differ in what they call the same thing. A person's login is `uid` on OpenLDAP and\n`sAMAccountName` on Active Directory, and group membership is `memberOf` on Active Directory but\noften `isMemberOf` elsewhere, so an attribute list is rarely portable between directories.\n\n## Searching large directories\n\nSearching is always paged, so a result set is not cut short by the maximum result size of the\ndirectory, which is 1000 entries on a default Active Directory installation. A search which\nreaches the configured maximum number of entries stops there and reports a warning saying so, and\na search which exceeds the configured time ends with an error from the directory.\n\nReferences to other directory servers are not followed unless this is asked for. Following one\nopens a new, anonymous connection to a server named by the directory rather than by this task,\nwhich commonly fails or stalls; Active Directory returns such a reference for every search below\nthe root of a domain.\n\n## Connecting securely\n\nAn `ldaps://` URL is encrypted from the start, and an `ldap://` URL can be upgraded before\nauthenticating where the directory offers no separate encrypted port. Certificates are validated\nin both cases. Where the directory uses an internal certificate authority, configure that\nauthority rather than switching validation off - a connection which accepts any certificate also\naccepts one presented by a machine in the middle, and the bind password is sent to it.\n\nLeaving both the bind name and the password empty binds anonymously, which many directories allow\nfor a limited part of their data.\n\n## Example searches\n\nAll people who have a mail address, below one organizational unit:\n\n```\n(&(objectClass=person)(mail=*))\n```\n\nThe enabled users of an Active Directory, without the disabled ones:\n\n```\n(&(objectCategory=person)(objectClass=user)(!(userAccountControl:1.2.840.113556.1.4.803:=2)))\n```\n\nThe members of one group, read from the group entry itself with a base scope:\n\n```\n(objectClass=groupOfNames)\n```\n", + "pluginIcon": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAyNCAyNCIgZmlsbD0ibm9uZSIKICAgICBzdHJva2U9ImN1cnJlbnRDb2xvciIgc3Ryb2tlLXdpZHRoPSIxLjYiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCIgc3Ryb2tlLWxpbmVqb2luPSJyb3VuZCI+CiAgPGNpcmNsZSBjeD0iNCIgY3k9IjUiIHI9IjIiLz4KICA8cGF0aCBkPSJNNCA3djEyIi8+CiAgPHBhdGggZD0iTTQgMTJoNSIvPgogIDxwYXRoIGQ9Ik00IDE5aDUiLz4KICA8Y2lyY2xlIGN4PSIxMSIgY3k9IjEyIiByPSIyIi8+CiAgPGNpcmNsZSBjeD0iMTEiIGN5PSIxOSIgcj0iMiIvPgogIDxjaXJjbGUgY3g9IjE4IiBjeT0iNi41IiByPSI0Ii8+CiAgPHBhdGggZD0iTTIwLjkgOS40IDIzIDExLjUiLz4KPC9zdmc+Cg==", + "properties": { + "ldap_url": { + "name": "ldap_url", + "title": "LDAP URL", + "description": "The URL of the directory server, e.g. `ldaps://directory.example.org:636`. Use `ldaps://` for an encrypted connection (port 636 by default), or `ldap://` for an unencrypted one (port 389 by default), which can be upgraded with the StartTLS parameter.", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "bind_dn": { + "name": "bind_dn", + "title": "Bind DN", + "description": "The distinguished name used to authenticate, e.g. `cn=reader,ou=services,dc=example,dc=org`. Leave empty together with the password to bind anonymously.", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "bind_password": { + "name": "bind_password", + "title": "Bind password", + "description": "The password of the bind DN. Leave empty together with the bind DN to bind anonymously.", + "type": "string", + "parameterType": "password", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "base_dn": { + "name": "base_dn", + "title": "Search base", + "description": "The distinguished name the search starts at, e.g. `ou=people,dc=example,dc=org`. An empty value is only accepted by directories which allow a search over all naming contexts.", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "search_filter": { + "name": "search_filter", + "title": "Search filter", + "description": "The LDAP filter which selects the entries, in the string representation of RFC 4515, e.g. `(&(objectClass=person)(mail=*))`.", + "type": "string", + "parameterType": "multiline string", + "value": "(objectClass=*)", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "scope": { + "name": "scope", + "title": "Scope", + "description": "How far below the search base the directory is searched.", + "type": "string", + "parameterType": "string", + "value": "SUBTREE", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "attributes": { + "name": "attributes", + "title": "Attributes", + "description": "The attributes to return, as a comma-separated list, e.g. `cn, mail, memberOf`. Each attribute becomes one value of the produced entities, in the order given here. `*` and `+` are not accepted.", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + } + }, + "properties_advanced": { + "start_tls": { + "name": "start_tls", + "title": "Use StartTLS", + "description": "If enabled, an `ldap://` connection is upgraded to an encrypted one before authenticating. Enable this where the directory offers encryption on the unencrypted port instead of a separate `ldaps://` endpoint. It cannot be combined with an `ldaps://` URL, which is encrypted already.", + "type": "string", + "parameterType": "boolean", + "value": "false", + "advanced": true, + "visibleInDialog": true, + "properties": {} + }, + "verify_certificates": { + "name": "verify_certificates", + "title": "Verify certificates", + "description": "If enabled, the certificate of the directory server is validated against the trusted certificate authorities. Disabling this accepts any certificate, including one presented by a machine-in-the-middle, and sends the bind password to it. Where the directory uses a private certificate authority, provide that authority below instead of disabling validation.", + "type": "string", + "parameterType": "boolean", + "value": "true", + "advanced": true, + "visibleInDialog": true, + "properties": {} + }, + "ca_certificate": { + "name": "ca_certificate", + "title": "Certificate authority", + "description": "The certificate authority which issued the certificate of the directory server, in PEM format. Needed only when that authority is not among the trusted ones, which is the usual case for a directory with an internal certificate authority.", + "type": "string", + "parameterType": "multiline string", + "value": "", + "advanced": true, + "visibleInDialog": true, + "properties": {} + }, + "follow_referrals": { + "name": "follow_referrals", + "title": "Follow referrals", + "description": "If enabled, references to other directory servers are followed. A referral is followed with a new, anonymous connection to a server named by the directory rather than by this task, which commonly fails or stalls. Enable it only for a directory whose data is genuinely partitioned across servers.", + "type": "string", + "parameterType": "boolean", + "value": "false", + "advanced": true, + "visibleInDialog": true, + "properties": {} + }, + "page_size": { + "name": "page_size", + "title": "Page size", + "description": "How many entries are fetched from the directory per request. This does not limit the number of entries returned overall, only how often the task talks to the directory while collecting them.", "type": "string", "parameterType": "Long", - "value": "1000", - "advanced": false, + "value": "500", + "advanced": true, "visibleInDialog": true, "properties": {} }, - "repos": { - "name": "repos", - "title": "Repositories", - "description": "Comma-separated list of repository names to search. Leave no trailing comma here. Example: 'windows,linux,firewall'. Leave empty to search all accessible repositories. Use the 'Preview repositories' action to see available options.", + "size_limit": { + "name": "size_limit", + "title": "Maximum entries", + "description": "The maximum number of entries to return, or 0 for no limit. When a search reaches this number, it stops and reports a warning, so that a subtree search over a large directory cannot fill a workflow unnoticed.", "type": "string", - "parameterType": "string", - "value": "", - "advanced": false, + "parameterType": "Long", + "value": "0", + "advanced": true, "visibleInDialog": true, "properties": {} }, - "paths_list": { - "name": "paths_list", - "title": "List of output paths", - "description": "Comma-separated list of field paths to include in output. Example: 'source_address, destination_address, user, log_ts'. Leave no trailing comma here. If specified, creates a fixed output schema with only these fields. Leave empty for automatic schema detection with all available fields. Use 'Preview output paths' action to discover available field names. See Logpoint field naming conventions at: https://docs.logpoint.com/docs/logpoint-taxonomy-guideline/en/latest/Field%20naming%20convention.html", + "time_limit": { + "name": "time_limit", + "title": "Time limit", + "description": "How many seconds the directory may spend on the search, or 0 to leave the decision to the directory.", "type": "string", - "parameterType": "string", - "value": "", - "advanced": false, + "parameterType": "Long", + "value": "0", + "advanced": true, "visibleInDialog": true, "properties": {} } }, - "properties_advanced": {}, - "actions": { - "preview_output_paths": { - "label": "Preview output paths", - "description": "Executes the configured query with a limit of 1 to retrieve and display available field paths in the result set. Use this to discover which fields you can specify in the 'List of output paths' parameter for schema customization.", - "icon": null - }, - "preview_repositories": { - "label": "Preview repositories", - "description": "Retrieves and displays all log repositories accessible with the configured credentials. Use this to identify repository names for the 'Repositories' parameter to filter searches to specific log sources.", - "icon": null - } - }, - "required": [ - "secret_key", - "query" - ], + "actions": {}, + "required": [], "distanceMeasureRange": null, "backendType": "python", "is_deprecated": false, @@ -6239,8 +7504,8 @@ "Uncategorized" ], "main_category": "Uncategorized", - "description": "Search for top-k metadata stored in Postgres Vector Store (PGVector).", - "markdownDocumentation": "\nThis workflow task search for the top-k metadata stored into Postgres Vector Store.\n\nThe incoming embedding entities are used to retrieve the nearest top-k\nvectors in the collection stored in the Postgres Vector Store.\nIt is possible to specify which paths are going to be used for searching as well as which Postgres\nVector Store and collection name.\n\nThe task uses the embeddings from the path configured with the Embedding Query Path\nparameter (`embedding_query_path`, default value: `_embedding`) to search over the collection.\nThe results are provided in the output path configured with the Search Result Path parameter\n(`search_result_path`, default value: `_search_result`).\n\nThe results in this output are structured like this:\n\n``` json\n[\n{\n \"id\": \"...\",\n \"metadata\": \"..\",\n \"_embedding_source\": \"..\",\n \"distance\": \"..\"\n}\n...\n]\n```\n", + "description": "Search for the top-k nearest matches to a query embedding in Postgres Vector Store (PGVector).", + "markdownDocumentation": "\nThis workflow task searches for the top-k nearest vectors stored in a collection in Postgres\nVector Store (PGVector).\n\nEach incoming entity supplies a query embedding on the single input path. The task compares it\nagainst the vectors in the configured collection and attaches the top-k nearest matches to the\nentity as a JSON array on an additional output path, together with each match's metadata, its\nstored embedding source, and its distance to the query.\n\nThe result is structured like this:\n\n``` json\n[\n{\n \"id\": \"...\",\n \"metadata\": \"..\",\n \"_embedding_source\": \"..\",\n \"distance\": \"..\"\n}\n...\n]\n```\n\nUse together with **Store Vector Embeddings**, which populates the collection this task searches.\n", "pluginIcon": "data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiIHN0YW5kYWxvbmU9Im5vIj8+CjxzdmcKICAgdmlld0JveD0iMCAwIDI0IDI0IgogICB2ZXJzaW9uPSIxLjEiCiAgIGlkPSJzdmc5MTkiCiAgIHNvZGlwb2RpOmRvY25hbWU9InBvc3RncmVzcWwuc3ZnIgogICBpbmtzY2FwZTp2ZXJzaW9uPSIxLjEuMiAoYjhlMjViZTgsIDIwMjItMDItMDUpIgogICB4bWxuczppbmtzY2FwZT0iaHR0cDovL3d3dy5pbmtzY2FwZS5vcmcvbmFtZXNwYWNlcy9pbmtzY2FwZSIKICAgeG1sbnM6c29kaXBvZGk9Imh0dHA6Ly9zb2RpcG9kaS5zb3VyY2Vmb3JnZS5uZXQvRFREL3NvZGlwb2RpLTAuZHRkIgogICB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciCiAgIHhtbG5zOnN2Zz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPgogIDxkZWZzCiAgICAgaWQ9ImRlZnM5MjMiIC8+CiAgPHNvZGlwb2RpOm5hbWVkdmlldwogICAgIGlkPSJuYW1lZHZpZXc5MjEiCiAgICAgcGFnZWNvbG9yPSIjZmZmZmZmIgogICAgIGJvcmRlcmNvbG9yPSIjNjY2NjY2IgogICAgIGJvcmRlcm9wYWNpdHk9IjEuMCIKICAgICBpbmtzY2FwZTpwYWdlc2hhZG93PSIyIgogICAgIGlua3NjYXBlOnBhZ2VvcGFjaXR5PSIwLjAiCiAgICAgaW5rc2NhcGU6cGFnZWNoZWNrZXJib2FyZD0iMCIKICAgICBzaG93Z3JpZD0iZmFsc2UiCiAgICAgaW5rc2NhcGU6em9vbT0iMTcuMTkyNjgxIgogICAgIGlua3NjYXBlOmN4PSIxMi4zNTk5MSIKICAgICBpbmtzY2FwZTpjeT0iMTIuODgzMzg5IgogICAgIGlua3NjYXBlOndpbmRvdy13aWR0aD0iMTkyMCIKICAgICBpbmtzY2FwZTp3aW5kb3ctaGVpZ2h0PSIxMDI3IgogICAgIGlua3NjYXBlOndpbmRvdy14PSIxNzI4IgogICAgIGlua3NjYXBlOndpbmRvdy15PSIyNSIKICAgICBpbmtzY2FwZTp3aW5kb3ctbWF4aW1pemVkPSIwIgogICAgIGlua3NjYXBlOmN1cnJlbnQtbGF5ZXI9InN2ZzkxOSIgLz4KICA8cGF0aAogICAgIGQ9Im0gMjEuMjk2MzQyLDE0LjExNTkxNSBhIDAuNDAyMDkyODQsMC4zODg1MTYyNSAwIDAgMCAtMC4wNDU4NCwtMC4wOTMyNCBRIDIxLjA4MTYyMywxMy43MTU3NDcgMjAuNDQwNjg3LDEzLjg0MzE4IGMgLTEuMzMwMTI0LDAuMjY0MTkxIC0xLjg0NDgwMiwwLjEwMTAxNSAtMi4wMzEzNzMsLTAuMDE1NTQgMS4wNzkyMTcsLTEuNTkxMzYzIDEuOTY2MjM0LC0zLjUxMzc0MSAyLjQ0NTUyOCwtNS4zMDcxMzIxIDAuMjE4NzM5LC0wLjgxNTg4NDIgMC42NDE3NCwtMi43Mzc0ODU1IDAuMDk4MTEsLTMuNjc1MzYzNyBBIDEuMjg2Njk3MSwxLjI0MzI1MiAwIDAgMCAyMC44MzIzMjcsNC42NjE3NjA1IEMgMTkuNzk0OTI4LDMuMzgyNzY1IDE4LjI3MjYwNCwyLjY5NTA5MTMgMTYuNDMxMDE5LDIuNjc2NDQyNSAxNS4yMjg3NjEsMi42NjQwMSAxNC4yMDM0MjQsMi45NDUyOTU3IDEzLjkyNTE3NiwzLjA0ODY0MTEgYSA4LjA0MTg1NjgsNy43NzAzMjQ5IDAgMCAwIC0wLjQxNDk2LC0wLjA2MzcxNyA2LjQzMzQ4NTQsNi4yMTYyNTk5IDAgMCAwIC0xLjA1NTA5MSwtMC4wOTg2ODMgQyAxMS41MDQ1NzcsMi44NzE0Nzc2IDEwLjY4MzUwNCwzLjA5MTM3NzggMTAuMDAyMzU4LDMuNTM4OTQ4NiA5LjMxMzk3NTUsMy4yODk1MjExIDYuMTUyNzIxNiwyLjI2MDczMDEgNC4xOTYxMzc5LDMuNjAwMzM0MSAzLjEwMTY0MTIsNC4zNDg2MTY0IDIuNTk4MjIwOSw1LjY4NTExMjMgMi42OTU1Mjc0LDcuNTc0MDc4MyAyLjcyODQ5OSw4LjIwOTY5MDkgMy4xMDMyNDk1LDEwLjE2NDcwNSAzLjY5NTEzMDIsMTIuMDM3MzUzIHEgMC41NTQ4ODgxLDEuNzU2MDkzIDEuMTUyMzk4MSwyLjc4MzMzIDAuNjY3NDc0MSwxLjE2MDExIDEuMzc4Mzc0MiwxLjM5MDg4OCBjIDAuMzYwMjc1MiwwLjExNTAwMSAwLjkxMTE0MjQsMC4xMTExMTYgMS40OTQxNzcsLTAuNTY2NDU2IGEgNDUuMDM0Mzk4LDQzLjUxMzgyIDAgMCAxIDEuNTY0MTQxMiwtMS43MTQxMzQgYyAwLjM0OTgyMDcsMC4xODI2MDMgMC43Mjg1OTIzLDAuMjgxMjg2IDEuMTE3ODE4MywwLjI5Mjk0MSB2IDAuMDAzMSBhIDguODQ2MDQyNSw4LjU0NzM1NzQgMCAwIDAgLTAuMTk4NjM0LDAuMjM2OTk1IGMgLTAuMjcyNjE5LDAuMzM0MTI0IC0wLjMyOTcxNjIsMC40MDQwNTcgLTEuMjA2Mjc4NiwwLjU3ODg5IC0wLjI0OTI5NzYsMC4wNDk3MyAtMC45MTE5NDY2LDAuMTgxMDQ4IC0wLjkyMTU5NjgsMC42MzAxNzMgYSAwLjQ4MjUxMTQxLDAuNDY2MjE5NSAwIDAgMCAwLjA3MzE4MSwwLjI1NDA5IGMgMC4xODI1NTAxLDAuMzI4Njg0IDAuNzQxNDU5MiwwLjQ3Mzk4OSAwLjgxNjI0ODUsMC40OTE4NjEgMS4wNzM1ODc5LDAuMjU4NzUyIDIuMDE0NDg0OSwwLjA3MTQ5IDIuNzExNzEzOSwtMC41Mjc2MDUgLTAuMDEzNjcsMS43MzM1NiAwLjA2MTkyLDMuNDMyOTMgMC4yNzc0NDQsMy45NTM1NDEgMC4xNzc3MjUsMC40Mjk2OTkgMC42MTI3OSwxLjQ3OTQ3IDEuOTg2MzM5LDEuNDc5NDcgcSAwLjMwMTU2OSw3Ljc3ZS00IDAuNjY2NjcsLTAuMDczMDQgYyAxLjQzMzA1OSwtMC4yOTY4MjYgMi4wNTU0OTgsLTAuOTA5MTI4IDIuMjk1OTUsLTIuMjU4MDU2IDAuMTIwNjI4LC0wLjY3NjAxOCAwLjMyMzI4MiwtMi4yMzM5NjkgMC40MzM0NTYsLTMuMTg2NjEgMC4wMTM2NywtMC4wNTQzOSAwLjAyODk1LC0wLjA5MzI0IDAuMDQ1ODQsLTAuMTA1Njc3IDAsMCAwLjA1NjI5LC0wLjAzNzMgMC4zNDMzODgsMC4wMjMzMSBsIDAuMDM1MzgsMC4wMDU0IDAuMjA0MjYzLDAuMDE3MDkgMC4wMTIwNiw3Ljc3ZS00IGMgMC42ODExNDUsMC4wMzAzIDEuNTM2Nzk5LC0wLjExMDMzOSAyLjAzNTM5NCwtMC4zMzQxMjQgMC41MTc4OTUsLTAuMjMzMTEgMS40NTIzNTksLTAuODAyNjc0IDEuMjgyNjc2LC0xLjI5NzY0NCBNIDQuMjU1NjQ3NiwxMS45MDM3MDMgQyAzLjY1NzMzMzUsMTAuMDExNjI5IDMuMzA4MzE2OSw4LjEwNzg5OTYgMy4yODA5NzQ2LDcuNTc0ODU1MyAzLjE5MzMxODMsNS44ODcxNDA3IDMuNjE2MzIsNC43MTMwNDQ3IDQuNTM3MTEyNiw0LjA4MzY0ODMgNi4wMTQ0MDE3LDMuMDc0MjgzMSA4LjQyOTM3MTMsMy42NjQwNTA4IDkuNDQ5MDc4NywzLjk4MjYzNDEgbCAtMC4wMDgwNCwwLjAwNzc3IEMgNy44MTQxNjkyLDUuNTc3MTA0OCA3Ljg1Mjc3MDIsOC4yOTA1MDIyIDcuODU4Mzk5NSw4LjQ1Njc4NzIgYyAwLDAuMDYzNzE3IDAuMDA0ODMsMC4xNTQ2Mjk1IDAuMDEyODY3LDAuMjc5NzMxNyAwLjAyNzM0MiwwLjQ1NTM0MSAwLjA4MDQxOSwxLjMwNTQxNDEgLTAuMDU5NTEsMi4yNjczODExIC0wLjEyODY2OTcsMC44OTM1ODcgMC4xNTYwMTIsMS43Njg1MjYgMC43ODI0NzI3LDIuNDAwMjUzIHEgMC4wOTY1MDIsMC4wOTc5MSAwLjIwMjY1NDcsMC4xODQxNTcgYyAtMC4yNzkwNTI0LDAuMjg4Mjc5IC0wLjg4NDYwNDIsMC45MjcgLTEuNTMwMzY1MywxLjY3NjgzNiAtMC40NTY3Nzc1LDAuNTI5OTM2IC0wLjc3MjAxODMsMC40MjgxNDUgLTAuODc0OTU0LDAuMzk0NzMyIEMgNi4wNzYzMjQsMTUuNTU4ODY0IDUuNzM3NzYxOCwxNS4yMDM3NiA1LjM5NTE3ODcsMTQuNjMyNjQxIDUuMDA5MTY5NiwxMy45ODA3MTEgNC42MjA3NDc5LDEzLjA1MzcxMSA0LjI1NzI1NiwxMS45MDM3MDMgbSA0LjgzMDc0MzQsMy45NTM1NDIgQSAxLjI4NjY5NzEsMS4yNDMyNTIgMCAwIDEgOC43NDA1OTExLDE1LjcxODkzMyBjIDAuMDcxNTczLC0wLjAzMDMgMC4xOTA1OTIxLC0wLjA2OTkzIDAuMzg4NDIxNywtMC4xMDg3ODUgMS4wMzI1NzQyLC0wLjIwNTkxMyAxLjE5MTgwMzIsLTAuMzUwNDQxIDEuNTQwMDE1MiwtMC43NzcwMzIgMC4wNzk2MSwtMC4wOTg2OCAwLjE2OTY4NCwtMC4yMDk3OTkgMC4yOTUxMzcsLTAuMzQ0MjI2IGEgMC4zMjE2NzQyNywwLjMxMDgxMyAwIDAgMCAwLjA1OTUxLC0wLjEwMTAxNCBjIDAuMTM2NzEyLC0wLjExNzMzMiAwLjIxODczOSwtMC4wODU0NyAwLjM1MDYyNSwtMC4wMzI2NCAwLjEyNTQ1MywwLjA1MDUxIDAuMjQ3Njg5LDAuMjAyMDI4IDAuMjk3NTQ5LDAuMzY5MDkgMC4wMjQxMywwLjA3OTI2IDAuMDQ5ODYsMC4yMjkyMjUgLTAuMDM2MTksMC4zNDU3OCAtMC43MjY5ODQsMC45ODM3MjMgLTEuNzg2OTAxLDAuOTcxMjkgLTIuNTQ3NjYwNiwwLjc4NzEzNCBtIDEuNjgzOTY0NiwtMy4wOTg4MDYgLTAuMDQxODIsMC4xMDg3ODUgYyAtMC4xMDY5NTcsMC4yNzc0IC0wLjIwNjY3NiwwLjUzNTM3NSAtMC4yNjg1OTgsMC43ODAxNCBDIDkuOTI1MTU2NywxMy42NDU4MSA5LjQwMjQzNiwxMy40MjM1NzkgOS4wMDU5NzI0LDEzLjAyMzQwNyA4LjUwMDk0MzgsMTIuNTE0NDUxIDguMjcxNzUwOSwxMS44MDY1NzQgOC4zNzYyOTUsMTEuMDgwODI2IDguNTIzNDYxLDEwLjA2NDQ2NyA4LjQ2OTU4MDYsOS4xNzk0Mjc0IDguNDQwNjI5OSw4LjcwMzg4MzUgTCA4LjQzMDE3NTUsOC41MzI5MzY0IEMgOC42NjgyMTQ0LDguMzI5MzUzOSA5Ljc2OTk0ODgsNy43NTkwMTIgMTAuNTU1NjM4LDcuOTMzMDY3MyBjIDAuMzU4NjY3LDAuMDc5MjU3IDAuNTc3NDA2LDAuMzE1NDc1MiAwLjY2NzQ3NCwwLjcyMTA4NjIgMC40NzA0NDksMi4xMDEwOTU1IDAuMDYyNzMsMi45NzYwMzQ1IC0wLjI2NTM4MSwzLjY4MDAyNTUgYSA3LjIzNzY3MTEsNi45OTMyOTI0IDAgMCAwIC0wLjE4NDk2MywwLjQyNDI2IG0gNS45MjIwMjQsMy41NTI1OTMgcSAtMC4wMTkzLDAuMjA2NjkgLTAuMDQ5ODYsMC40NjMxMTEgbCAtMC4xMTc0MTEsMC4zNDAzNCBhIDAuMzIxNjc0MjcsMC4zMTA4MTMgMCAwIDAgLTAuMDE0NDgsMC4wODM5MiBjIC0wLjAwNDgsMC4zNjkwOSAtMC4wNDM0MywwLjUwNDI5NCAtMC4wOTI0OCwwLjY3NjAxOCAtMC4wNTE0NywwLjE3Nzk0IC0wLjEwODU2NSwwLjM3OTE5MiAtMC4xNDQ3NTMsMC44MjEzMjMgLTAuMDg4NDYsMS4wOTg3MjQgLTAuNzA2MDc1LDEuNzMwNDUyIC0xLjk0MzcxNywxLjk4NjA5NSAtMS4yMTgzNDEsMC4yNTI1MzYgLTEuNDM0NjY3LC0wLjM4NTQwOCAtMS42MjQ0NTUsLTAuOTQ4NzU2IGEgNS42MjkyOTk4LDUuNDM5MjI3NCAwIDAgMCAtMC4wNjI3MywtMC4xNzYzODcgYyAtMC4xNzI5LC0wLjQ1NTM0MSAtMC4xNTI3OTYsLTEuMDk3MTcgLTAuMTI2MjU4LC0xLjk4NTMxOCAwLjAxMjg3LC0wLjQzNTkxNSAtMC4wMjAxLC0xLjQ3NzEzOCAtMC4yNjUzODEsLTIuMDU2MDI4IHEgMC4wMDQ4LC0wLjM0MTg5NCAwLjAxNTI4LC0wLjY5MzExMyBhIDAuMzIxNjc0MjcsMC4zMTA4MTMgMCAwIDAgLTAuMDEyODcsLTAuMDg3OCAxLjYwODM3MTQsMS41NTQwNjUgMCAwIDAgLTAuMDM1MzgsLTAuMTYxNjIzIGMgLTAuMDk4MTEsLTAuMzMyNTcgLTAuMzM3NzU4LC0wLjYxMDc0OCAtMC42MjcyNjUsLTAuNzI2NTI1IC0wLjExNDE5NSwtMC4wNDU4NCAtMC4zMjQwODcsLTAuMTI5NzY1IC0wLjU3NjYwMSwtMC4wNjc2IDAuMDUzODgsLTAuMjE0NDYxIDAuMTQ3MTY2LC0wLjQ1NjExOCAwLjI0ODQ5MywtMC43MTg3NTUgbCAwLjA0MjYyLC0wLjExMDMzOSBjIDAuMDQ4MjUsLTAuMTI0MzI1IDAuMTA3NzYxLC0wLjI1MjUzNSAwLjE3MTI5MSwtMC4zODg1MTYgMC4zNDI1ODMsLTAuNzM2NjI3IDAuODEyMjI4LC0xLjc0NTIxNSAwLjMwMjM3NCwtNC4wMjM0NzQ0IEMgMTEuNTkwNjI1LDcuNjg0NDE2OSAxMC45NTI5MDYsNy4yNjc5Mjc1IDkuOTg2Mjc0OCw3LjM2NDI3OTUgOS40MDcyNjExLDcuNDIyNTU3IDguODc2NDk4NSw3LjY0ODY3MzQgOC42MTE5MjE0LDcuNzc3NjYwOCBBIDQuODI1MTE0MSw0LjY2MjE5NSAwIDAgMCA4LjQ1NDMwMSw3Ljg1ODQ3MjIgQyA4LjUyODI4NjEsNi45OTkwNzQyIDguODA3MzM4Niw1LjM5MjE3MSA5Ljg1MDM2NzQsNC4zNzU4MTI2IGEgMy4yMTY3NDI3LDMuMTA4MTMgMCAwIDEgMC4yNDM2Njg2LC0wLjIxNDQ2MSAwLjI4MTQ2NDk5LDAuMjcxOTYxMzcgMCAwIDAgMC4xMTY2MDcsLTAuMDQ5NzMgQyAxMC44MTUzOSwzLjY2ODcxMyAxMS41NzM3MzcsMy40NTExNDM5IDEyLjQ2Mzk3MSwzLjQ2NDM1MzQgcSAwLjQ5NTM3OCwwLjAwNzc3IDAuOTQ0MTE0LDAuMDYyOTQgYyAxLjU2MDEyLDAuMjc1ODQ2NSAyLjYwODc3OCwxLjEyNDM2NiAzLjI0NTY5MywxLjg1MTY2ODQgMC42NTQ2MDcsMC43NDc1MDUzIDEuMDA5MjUzLDEuNTAwNDQ5NyAxLjE1MDc5LDEuOTA2ODM3NyAtMS4wNjM5MzgsLTAuMTA0MTIyMyAtMS43ODc3MDUsMC4wOTg2ODMgLTIuMTU1MjE4LDAuNjA2MDg1NCAtMC43OTc3NTIsMS4xMDE4MzIgMC40Mzc0NzcsMy4yNDE3NzkxIDEuMDMwOTY2LDQuMjcwNTcwMSAwLjEwODU2NSwwLjE4ODA0MiAwLjIwMjY1NSwwLjM1MTIxOSAwLjIzMjQxLDAuNDE5NTk4IDAuMTkzMDA1LDAuNDUzMDEgMC40NDMxMDYsMC43NTUyNzUgMC42MjU2NTYsMC45NzU5NTMgMC4wNTYyOSwwLjA2NzYgMC4xMTA5NzgsMC4xMzI4NzIgMC4xNTE5OTIsMC4xOTAzNzIgLTAuMzIxNjc1LDAuMDkwMTQgLTAuOTAwNjg4LDAuMjk3NjA0IC0wLjg0ODQxNiwxLjMzNDE2NSAtMC4wMTA0NiwwLjEyMTIxNyAtMC4wMzQ1OCwwLjM0NzMzNCAtMC4wNjc1NSwwLjYzMzI4MiAtMC4wMzY5OSwwLjE2MTYyMiAtMC4wNTYyOSwwLjM1NzQzNSAtMC4wODA0MiwwLjU5NTIwNyBtIDAuNzE1NzI1LC0xLjI1OTU3IGMgLTAuMDMyMTcsLTAuNjQ2NDkxIDAuMjE3MTMsLTAuNzE0MDkzIDAuNDgwMDk5LC0wLjc4NDgwMyBsIDAuMTA4NTY1LC0wLjAzMTg2IGEgMC44MDQxODU2OCwwLjc3NzAzMjQ5IDAgMCAwIDAuMTA3NzYxLDAuMDgwMDMgYyAwLjQ1ODM4NSwwLjI5MjE2NCAxLjI3MzAyNiwwLjMyNzEzMSAyLjQxODE4NiwwLjEwNDEyMyAtMC4xNjI0NDYsMC4xMzc1MzQgLTAuNDE3MzcyLDAuMzEwODEzIC0wLjc2NjM4OSwwLjQ2Njk5NiAtMC4zMjk3MTYsMC4xNDc2MzYgLTAuODgxMzg4LDAuMjU4NzUyIC0xLjQwNDkxMiwwLjI4Mjg0IC0wLjU3OTAxNCwwLjAyNjQyIC0wLjg3MzM0NiwtMC4wNjIxNiAtMC45NDMzMSwtMC4xMTczMzIgbSAwLjQ1ODM4NiwtNy4yMDM4NjggYyAtMC4wMDQ4LDAuMjcxOTYxNCAtMC4wNDM0MywwLjUxOTgzNDggLTAuMDg0NDQsMC43Nzc4MDk1IC0wLjA0NDIzLDAuMjc4MTc3NyAtMC4wOTAwNywwLjU2NTY3OTcgLTAuMTAyMTMxLDAuOTE0NTY3MyAtMC4wMTEyNiwwLjMzODc4NjIgMC4wMzIxNywwLjY5MTU1ODIgMC4wNzQ3OSwxLjAzMzQ1MzIgMC4wODYwNSwwLjY4OTIyOCAwLjE3MzcwNCwxLjM5ODY1OCAtMC4xNjY0NjcsMi4wOTg3NjQgYSAzLjIxNjc0MjcsMy4xMDgxMyAwIDAgMSAtMC4xNTExODcsLTAuMjk5MTU3IDYuNDMzNDg1NCw2LjIxNjI1OTkgMCAwIDAgLTAuMjYxMzYsLTAuNDc5NDI5IEMgMTYuNjgxOTI2LDExLjAzNTc1MiAxNS41MjMwOTQsOS4wMjYzNDYzIDE2LjExNTc3OSw4LjIwNzM1NDEgMTYuNDIxMzcsNy43ODU0MjU0IDE3LjE5NDk5Niw3Ljc2NzU1MzcgMTcuODY4MSw3Ljg0NzU4OCBtIDAuMTgzMzU0LDUuNDQ5MzI5IC0wLjA2ODM2LC0wLjA4MzE0IC0wLjAyODE1LC0wLjAzNDE5IGMgMC41ODM4MzgsLTAuOTMyNDM5IDAuNDY5NjQ0LC0xLjg1NDc3NyAwLjM2NzUxMywtMi42NzIyMTUgLTAuMDQxODIsLTAuMzM1Njc4IC0wLjA4MDQyLC0wLjY1MjcwNyAtMC4wNzA3NywtMC45NDk1MzM1IDAuMDEwNDUsLTAuMzE2MjUyMiAwLjA1MzA4LC0wLjU4NjY1OTUgMC4wOTQ4OSwtMC44NDg1MTk0IDAuMDUxNDcsLTAuMzIyNDY4NSAwLjEwNDU0NCwtMC42NTU4MTU1IDAuMDg5MjYsLTEuMDQ4OTkzOSBhIDAuNDgyNTExNDEsMC40NjYyMTk1IDAgMCAwIDAuMDA5NywtMC4xNDc2MzYyIEMgMTguNDA4NTExLDcuMTM1MDU0OSAxNy45NjI5OTMsNi4wMDY4MDM4IDE3LjA1NDI2Myw0Ljk4NTAwNiBhIDYuMjcyNjQ4Myw2LjA2MDg1MzQgMCAwIDAgLTIuMTYxNjUxLC0xLjU4NTE0NjMgNy40Nzg5MjY4LDcuMjI2NDAyMiAwIDAgMSAxLjYyNjg2NywtMC4xNDQ1MjggYyAxLjY1MDE4OSwwLjAzNTc0MyAyLjk1NTM4MywwLjYzMjUwNDQgMy44NzkzOTIsMS43NzM5NjUyIGEgMC44MDQxODU2OCwwLjc3NzAzMjQ5IDAgMCAxIDAuMDUzODgsMC4wNzc3MDMgQyAyMS4wMzQxNzgsNi4xNjA2NTYyIDIwLjIzMDc5Niw5Ljk4Mjg3OSAxOC4wNTA2NDksMTMuMjk2OTIzIE0gMTAuOTYwOTQ4LDguNTQ0NTkxOSBDIDEwLjk0MDg0Myw4LjY4NDQ1NzcgMTAuNzExNjUsOC44NzMyNzY2IDEwLjQ2MTU0OSw4Ljg3MzI3NjYgQSAwLjgwNDE4NTY4LDAuNzc3MDMyNDkgMCAwIDEgMTAuMzk2NDA5LDguODY4NjE0NCAwLjY0MzM0ODU0LDAuNjIxNjI1OTkgMCAwIDEgOS45ODk0OTE1LDguNjIzODQ5MiBDIDkuOTUyNDk5LDguNTc3MjI3MiA5Ljg5Mjk4OTIsOC40ODU1Mzc0IDkuOTA0MjQ3OCw4LjQwMjM5NDkgYSAwLjE3NjkyMDg1LDAuMTcwOTQ3MTUgMCAwIDEgMC4wNzQ3ODksLTAuMTE1Nzc3OCBjIDAuMDk0ODk0LC0wLjA2OTE1NiAwLjI4MzA3MzIsLTAuMDk0Nzk4IDAuNDkwNTUzMiwtMC4wNjY4MjUgMC4yNTQxMjMsMC4wMzQxODkgMC41MTYyODgsMC4xNDk5NjczIDAuNDkwNTU0LDAuMzI0Nzk5NiBtIDYuMzc3MTkyLC0wLjMxOTM2MDQgYyAwLjAwODgsMC4wNjIxNjMgLTAuMDM5NDEsMC4xNTU0MDY1IC0wLjEyMzA0LDAuMjQwODgwMSBhIDAuNTc5MDEzNjksMC41NTk0NjMzOSAwIDAgMSAtMC4zMjgxMDgsMC4xNzMyNzgyIDAuODA0MTg1NjgsMC43NzcwMzI0OSAwIDAgMSAtMC4wNjAzMSwwLjAwMzg5IGMgLTAuMjM1NjI2LDAgLTAuNDM1MDY0LC0wLjE4MTgyNTYgLTAuNDUwMzQ0LC0wLjI4ODI3OTEgLTAuMDE5MywtMC4xMzc1MzQ3IDAuMjEyMzA1LC0wLjI0MDg4IDAuNDUwMzQ0LC0wLjI3MzUxNTQgMC4yMzk2NDcsLTAuMDMyNjM1IDAuNDkyMTYyLDAuMDA2OTkgMC41MTE0NjIsMC4xNDM3NTEiCiAgICAgaWQ9InBhdGg5MTciCiAgICAgc3R5bGU9InN0cm9rZS13aWR0aDowLjc5MDQ5MyIgLz4KPC9zdmc+Cg==", "properties": { "host": { @@ -6301,7 +7566,7 @@ "collection_name": { "name": "collection_name", "title": "Collection Name", - "description": "The name of the collection that will be used for search.", + "description": "The name of the collection in Postgres Vector Store (PGVector).", "type": "string", "parameterType": "string", "value": null, @@ -6323,7 +7588,7 @@ "embedding_query_path": { "name": "embedding_query_path", "title": "Embedding Query Path", - "description": "The path containing the embedding to be used for searching.", + "description": "The path containing the query embedding to search with. Its value must be a string representation of a list of floats, e.g. `[0.1, 0.2, ...]`; the task fails if it cannot be parsed as one.", "type": "string", "parameterType": "string", "value": "_embedding", @@ -6347,7 +7612,7 @@ "distance_strategy": { "name": "distance_strategy", "title": "Distance Strategy", - "description": "The distance strategy to use. (default: COSINE)", + "description": "The distance strategy used to rank search matches.", "type": "string", "parameterType": "enumeration", "value": "COSINE", @@ -6420,7 +7685,7 @@ ], "main_category": "Uncategorized", "description": "Sends an email using an SMTP server.", - "markdownDocumentation": "Sends an email using an SMTP server with support for both plain text and HTML formatted messages.\n\n## Features\n\n- Send emails with optional file attachments from workflow datasets\n- Support for multiple recipients (To, CC, BCC)\n- HTML and plain text message formatting\n- SSL/TLS connection support\n- Configurable retry mechanism with delays\n- Dynamic email configuration via input parameters\n\n## Usage\n\nWhen connected to a dataset that is based on a file in a workflow, it will send that file as an attachment whenever\nthe workflow is executed. This allows you to send workflow results via email automatically.\n\n## HTML email example\n\nEnable HTML formatting and use standard HTML markup in your message:\n\n```html\n\n\n\n Workflow Results\n\n\n\n

Workflow Results

\n\n

Here are the results of your workflow:

\n
    \n
  • Total records processed: 1,234
    • \n
  • Status: Success
    • \n
\n

Please find the detailed report attached.

\n\n\n \n \n \n
MetricValue
Processing Time2.5 minutes
Error Rate0%
\n\n\n\n```\n", + "markdownDocumentation": "Sends an email using an SMTP server with support for both plain text and HTML formatted messages.\n\n## Features\n\n- Send emails with optional file attachments from workflow datasets\n- Support for multiple recipients (To, CC, BCC)\n- HTML and plain text message formatting\n- SSL/TLS connection support\n- Configurable retry mechanism with delays\n- Dynamic email configuration via input parameters\n\n## Usage\n\nWhen connected to a dataset that is based on a file in a workflow, it will send that file as an attachment whenever\nthe workflow is executed. This allows you to send workflow results via email automatically.\n\n## SSL/TLS\n\nThe connection is upgraded with STARTTLS if the server offers it. \"Force SSL\" uses SSL/TLS from the start.\nThe certificate of the server must be trusted by the Java runtime and match the configured host.\nThe host name check can be switched off with the advanced parameter \"Verify server host name\".\n\n## HTML email example\n\nEnable HTML formatting and use standard HTML markup in your message:\n\n```html\n\n\n\n Workflow Results\n\n\n\n

Workflow Results

\n\n

Here are the results of your workflow:

\n
    \n
  • Total records processed: 1,234
    • \n
  • Status: Success
    • \n
\n

Please find the detailed report attached.

\n\n\n \n \n \n
MetricValue
Processing Time2.5 minutes
Error Rate0%
\n\n\n\n```\n", "pluginIcon": "data:image/svg+xml;base64,PHN2ZyBpZD0iaWNvbiIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiBmaWxsPSJjdXJyZW50Q29sb3IiIHZpZXdCb3g9IjAgMCAzMiAzMiI+PGRlZnM+PHN0eWxlPi5jbHMtMXtmaWxsOm5vbmU7fTwvc3R5bGU+PC9kZWZzPjx0aXRsZT5lbWFpbDwvdGl0bGU+PHBhdGggZD0iTTI4LDZINEEyLDIsMCwwLDAsMiw4VjI0YTIsMiwwLDAsMCwyLDJIMjhhMiwyLDAsMCwwLDItMlY4QTIsMiwwLDAsMCwyOCw2Wk0yNS44LDgsMTYsMTQuNzgsNi4yLDhaTTQsMjRWOC45MWwxMS40Myw3LjkxYTEsMSwwLDAsMCwxLjE0LDBMMjgsOC45MVYyNFoiIHRyYW5zZm9ybT0idHJhbnNsYXRlKDApIi8+PHJlY3QgaWQ9Il9UcmFuc3BhcmVudF9SZWN0YW5nbGVfIiBkYXRhLW5hbWU9IiZsdDtUcmFuc3BhcmVudCBSZWN0YW5nbGUmZ3Q7IiBjbGFzcz0iY2xzLTEiIHdpZHRoPSIzMiIgaGVpZ2h0PSIzMiIvPjwvc3ZnPg==", "properties": { "host": { @@ -6611,6 +7876,17 @@ "advanced": true, "visibleInDialog": true, "properties": {} + }, + "verifyServerHostName": { + "name": "verifyServerHostName", + "title": "Verify server host name", + "description": "When enabled, the certificate of the SMTP server must match the configured host for SSL/TLS connections. Disable this only if the server cannot present a matching certificate, because the connection can then be intercepted.", + "type": "string", + "parameterType": "boolean", + "value": "true", + "advanced": true, + "visibleInDialog": true, + "properties": {} } }, "actions": {}, @@ -6633,8 +7909,8 @@ "Uncategorized" ], "main_category": "Uncategorized", - "description": "Send messages to Mattermost channels and/or users.", - "markdownDocumentation": "This task sends messages to Mattermost channels and users.\n\nYou need a bot account in order to connect to Mattermost.\nLearn more on bot accounts at\n[developers.mattermost.com](https://developers.mattermost.com):\n\n- [Using bot accounts](https://developers.mattermost.com/integrate/reference/bot-accounts/)\n- [Personal access tokens](https://developers.mattermost.com/integrate/reference/personal-access-token/)\n\nThe task has two working modes.\n\n# Single Message\n\nYou can send a single static message to a pre-configured channel or user.\nJust configure the User and/or Channel and Message parameters to do so.\n\n# Multiple Messages\n\nYou can send multiple messages to different channels or users by piping data into\nthe task. For each entity, a message is send. For dynamic messages the following\ninput paths are recognized:\n\n- user\n- channel\n- message\n", + "description": "Post messages to Mattermost users and channels through a bot account.", + "markdownDocumentation": "This task posts messages to Mattermost. Every message is sent by a bot\naccount, so recipients see it as coming from the bot rather than from the\nperson who ran the workflow. You need that bot account and a personal access\ntoken for it before the task can do anything - learn more at\n[developers.mattermost.com](https://developers.mattermost.com):\n\n- [Using bot accounts](https://developers.mattermost.com/integrate/reference/bot-accounts/)\n- [Personal access tokens](https://developers.mattermost.com/integrate/reference/personal-access-token/)\n\n# Ports\n\nOne input port accepts entities carrying the paths `user`, `channel` and\n`message`. Each entity is sent as one message per recipient it names: a direct\nmessage to the user, a post to the channel, or both.\n\nThere is no output port. The task is a terminal step - it hands nothing on to\nthe rest of the workflow and reports only how many messages went where.\n\nA recipient and a message configured on the task itself are sent once, before\nany entity is read. Configuring them while entities also arrive sends both:\nthat message once, and one message per entity on top of it.\n\n# Caveats\n\n- One field names one recipient, matched exactly. A user is found by username,\n nickname, email address or full name; a channel by its name or its display\n name. A comma separated list is **not** split into several recipients - it is\n looked up as a single name, which fails.\n- What an entity carries is not merged with what is configured on the task. An\n entity without a message does not fall back to the configured message; it\n fails.\n- The task stops at the first entity it cannot send - one naming no recipient,\n one carrying no message, or one naming a recipient Mattermost does not know.\n Messages sent for earlier entities have already been posted and are not taken\n back.\n- Requests to Mattermost time out after two seconds, so an unreachable or slow\n server fails the task rather than delaying it.\n", "pluginIcon": "data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiIHN0YW5kYWxvbmU9Im5vIj8+CjwhLS0gQ3JlYXRlZCB3aXRoIElua3NjYXBlIChodHRwOi8vd3d3Lmlua3NjYXBlLm9yZy8pIC0tPgoKPHN2ZwogICB3aWR0aD0iOTAuODU4NjA0bW0iCiAgIGhlaWdodD0iOTcuMjEwOG1tIgogICB2aWV3Qm94PSIwIDAgOTAuODU4NjA1IDk3LjIxMDgwMSIKICAgdmVyc2lvbj0iMS4xIgogICBpZD0ic3ZnNSIKICAgc29kaXBvZGk6ZG9jbmFtZT0iY3VzdG9tLnN2ZyIKICAgaW5rc2NhcGU6dmVyc2lvbj0iMS4xLjIgKGI4ZTI1YmU4LCAyMDIyLTAyLTA1KSIKICAgeG1sbnM6aW5rc2NhcGU9Imh0dHA6Ly93d3cuaW5rc2NhcGUub3JnL25hbWVzcGFjZXMvaW5rc2NhcGUiCiAgIHhtbG5zOnNvZGlwb2RpPSJodHRwOi8vc29kaXBvZGkuc291cmNlZm9yZ2UubmV0L0RURC9zb2RpcG9kaS0wLmR0ZCIKICAgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIgogICB4bWxuczpzdmc9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KICA8c29kaXBvZGk6bmFtZWR2aWV3CiAgICAgaWQ9Im5hbWVkdmlldzgzNiIKICAgICBwYWdlY29sb3I9IiNmZmZmZmYiCiAgICAgYm9yZGVyY29sb3I9IiM2NjY2NjYiCiAgICAgYm9yZGVyb3BhY2l0eT0iMS4wIgogICAgIGlua3NjYXBlOnBhZ2VzaGFkb3c9IjIiCiAgICAgaW5rc2NhcGU6cGFnZW9wYWNpdHk9IjAuMCIKICAgICBpbmtzY2FwZTpwYWdlY2hlY2tlcmJvYXJkPSIwIgogICAgIGlua3NjYXBlOmRvY3VtZW50LXVuaXRzPSJtbSIKICAgICBzaG93Z3JpZD0iZmFsc2UiCiAgICAgaW5rc2NhcGU6em9vbT0iMC44MDY1NDMwMyIKICAgICBpbmtzY2FwZTpjeD0iMjY0LjcwOTk5IgogICAgIGlua3NjYXBlOmN5PSIyNzguMzQ4NDUiCiAgICAgaW5rc2NhcGU6d2luZG93LXdpZHRoPSIxOTIwIgogICAgIGlua3NjYXBlOndpbmRvdy1oZWlnaHQ9IjEwMjciCiAgICAgaW5rc2NhcGU6d2luZG93LXg9IjE3MjgiCiAgICAgaW5rc2NhcGU6d2luZG93LXk9IjI1IgogICAgIGlua3NjYXBlOndpbmRvdy1tYXhpbWl6ZWQ9IjAiCiAgICAgaW5rc2NhcGU6Y3VycmVudC1sYXllcj0ic3ZnNSIKICAgICBmaXQtbWFyZ2luLXRvcD0iMTAiCiAgICAgZml0LW1hcmdpbi1sZWZ0PSIxMCIKICAgICBmaXQtbWFyZ2luLXJpZ2h0PSIxMCIKICAgICBmaXQtbWFyZ2luLWJvdHRvbT0iMTAiIC8+CiAgPGRlZnMKICAgICBpZD0iZGVmczIiIC8+CiAgPHBhdGgKICAgICBkPSJNIDc2Ljg4NDMwMiwzMy4wNDgwMTggQSAzMy4xOTI1NDksMzMuMTkyNTQ5IDAgMCAwIDY3LjIyMzUyMywyMC44NTQxMjQgbCAwLjM0OTM5Niw3LjE5NzU0MSAwLjM4NDMzNCwwLjUwNjYyNCBhIDI0LjY4NDc3NCwyNC42ODQ3NzQgMCAwIDEgNC40MTk4NSwxOS44NDU2NDkgMjUuMjk2MjE2LDI1LjI5NjIxNiAwIDAgMSAtMzEuMjcwODc0LDE5LjY4ODQyMyBDIDI4LjU2MjkzOSw2NC45NDc4MDQgMjAuMTc3NDU0LDUyLjA5MDA1OSAyMi41NTMzNDEsMzkuMzcyMDcyIDI0LjQyMjYwNiwyOS4zMjY5NTkgMzAuMzYyMzI1LDIyLjYzNjA0IDQwLjAwNTYzNCwxOS4yNDY5MDYgbCAwLjM0OTM5NSwtMC4xMzk3NTggMC4yNzk1MTYsLTAuMjA5NjM4IDMuNTExNDIzLC01LjA2NjIzMSBhIDMzLjM0OTc3NiwzMy4zNDk3NzYgMCAwIDAgLTMwLjk1NjQyLDMwLjI0MDE1OSAzMy40NTQ1OTUsMzMuNDU0NTk1IDAgMCAwIDI2LjYyMzkxOCwzNi4wMDUxOCAzMy40MDIxODYsMzMuNDAyMTg2IDAgMCAwIDM4Ljc4Mjg3MiwtMjMuMDYwMDg2IGMgMi40OTgxNzcsLTguMTU4Mzc5IDEuOTIxNjc1LC0xNi4yMTE5NCAtMS42OTQ1NjYsLTIzLjk4NTk4NCB6IE0gMzUuNjU1NjYzLDQzLjgyNjg2MSBBIDEyLjE3NjQyNCwxMi4xNzY0MjQgMCAwIDAgNDkuMzY5NDI2LDU0LjcyNzk5MyAxMi4yMjg4MzQsMTIuMjI4ODM0IDAgMCAwIDU5LjkwMzY5Myw0MC41MjUwNzYgQyA1OC45MDc5MTcsMzUuMjQ5MjA4IDU3Ljc4OTg1MiwyOS45OTA4MDkgNTYuNzI0MTk2LDI0LjczMjQxMSBsIC0xLjM2MjY0MSwtNi42NTU5NzkgLTAuMzQ5Mzk1LDAuMjk2OTg2IC03LjAyMjg0NSw2LjkwMDU1NSAtOC41NjAxODMsOC4zODU0ODYgYSAxMS44OTY5MDgsMTEuODk2OTA4IDAgMCAwIC0zLjc1NTk5OSwxMC4xNjc0MDIgeiIKICAgICBpZD0icGF0aDkxOSIKICAgICBzdHlsZT0ic3Ryb2tlLXdpZHRoOjAuMTc0Njk3IiAvPgo8L3N2Zz4K", "properties": { "url": { @@ -6662,55 +7938,344 @@ "bot_name": { "name": "bot_name", "title": "Bot name", - "description": "The name or display name of the bot you want to use to connect.", + "description": "The bot account that sends the messages, named by its username, nickname, email address or full name.", + "type": "string", + "parameterType": "string", + "value": null, + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "user": { + "name": "user", + "title": "User", + "description": "The single user account that receives the message as a direct message, named by username, nickname, email address or full name. Type at least one letter to search for users. Suggestions need the URL and the Access Token to be filled in first.", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "channel": { + "name": "channel", + "title": "Channel", + "description": "The single channel that receives the message, named by its name or its display name. Type at least one letter to search for channels. Suggestions need the URL and the Access Token to be filled in first.", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "message": { + "name": "message", + "title": "Message", + "description": "The message text, interpreted by Mattermost as Markdown. Its maximum length is a server setting, 16383 characters by default.", + "type": "string", + "parameterType": "multiline string", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + } + }, + "properties_advanced": {}, + "actions": {}, + "required": [ + "url", + "access_token", + "bot_name" + ], + "distanceMeasureRange": null, + "backendType": "python", + "is_deprecated": false, + "tags": [ + "WorkflowTask", + "PythonPlugin" + ], + "pluginType": "customtask", + "relatedPlugins": [] + }, + "cmem_plugin_ntfy-SendAttachment": { + "pluginId": "cmem_plugin_ntfy-SendAttachment", + "title": "Send ntfy attachment", + "categories": [ + "Uncategorized" + ], + "main_category": "Uncategorized", + "description": "Send a file as an attachment to a topic on an ntfy server.", + "markdownDocumentation": "## What it does\n\nSends files to an ntfy topic as attachments, so a report a workflow just\nproduced arrives where somebody will see it.\n\n## What flows through the ports\n\nOne input port accepts file entities - from a file dataset, a project resource\nor a preceding task that produces files - and every file that arrives is sent\nas one notification, byte for byte, under its own name. There is no output\nport: the task is a terminal step and hands nothing on.\n\n## Where it sits\n\nIt is the file sending counterpart of **Send ntfy message**, which sends text,\nand of **Get ntfy attachments**, which fetches files in the other direction.\n\n## Caveats\n\n- **The file is the body of the notification**, so the text fields configured\n here travel beside it and describe it rather than replacing it.\n- **An input is required.** With nothing connected the task has no file to\n send and fails instead of notifying anybody.\n- **ntfy limits an attachment to 15 MB** by default and keeps it for three\n hours, after which the recipient's link stops working. A file the server\n refuses fails the run, and files already sent are not taken back.\n- **The media type a recipient sees comes from the file name.** ntfy reads it\n from the name's extension and ignores what the request declares, so a file\n entity's own media type does not travel with it and a file arriving without\n an extension is offered as a generic download.\n- **An upload is never retried, not even when the server asks for a slower\n pace.** A file arrives as a stream that can be read once, so a second attempt\n would upload an empty attachment that ntfy accepts and stores. Failing on the\n rate limit and re-running the workflow is the honest outcome.\n- **What is configured applies to every file.** A per-file title or priority\n is not read from the entities, which carry a file and nothing else.\n- **An unprotected topic is public.** A topic needs no sign-up and carries no\n password of its own, so on a server without access control anybody who\n guesses the name can download every file this task sends - and a file is\n usually a whole report rather than one line of text. Give the topic a long\n random suffix rather than a word like `reports`, and protect it with the\n server's own [access control](https://docs.ntfy.sh/config/#access-control)\n where that is available.\n", + "pluginIcon": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIyNCIgaGVpZ2h0PSIyNCIgdmlld0JveD0iMCAwIDI0IDI0IiBmaWxsPSJjdXJyZW50Q29sb3IiIGZpbGwtcnVsZT0iZXZlbm9kZCIgZm9jdXNhYmxlPSJmYWxzZSIgYXJpYS1oaWRkZW49InRydWUiPgo8ZyB0cmFuc2Zvcm09InRyYW5zbGF0ZSgtMS42IDEuMSkgc2NhbGUoMC43OCkiPjxwYXRoIGQ9Ik02IDIuMmg3LjNMMTkuNiA4LjV2MTMuM0g2eiIvPjxwYXRoIGQ9Ik0xNC4yIDIuNHY1LjJoNS4yeiIvPjwvZz48cGF0aCBkPSJNMTkuNiA3LjYgMjMgMTIuMWgtMi4xdjYuM2gtMi42di02LjNIMTZ6Ii8+Cjwvc3ZnPgo=", + "properties": { + "server": { + "name": "server", + "title": "Server", + "description": "The base URL of the ntfy server. The default is the public service; a self-hosted ntfy is named here instead, for example https://ntfy.example.org - see [installing ntfy](https://docs.ntfy.sh/install/).", + "type": "string", + "parameterType": "string", + "value": "https://ntfy.sh", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "access_token": { + "name": "access_token", + "title": "Access token", + "description": "The access token to authenticate with. A public topic needs none, so this can be left empty. See [access tokens](https://docs.ntfy.sh/config/#access-tokens).", + "type": "string", + "parameterType": "password", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "topic": { + "name": "topic", + "title": "Topic", + "description": "The topic the files are sent to. A topic needs no sign-up and has no password of its own, so give it a long random suffix nobody can guess rather than a plain word - see [picking a topic](https://docs.ntfy.sh/publish/#picking-a-topic).", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "title": { + "name": "title", + "title": "Title", + "description": "The title shown with every attachment of this task. It travels as a header, so it is a single line and a line break becomes a space. See [message titles](https://docs.ntfy.sh/publish/#message-title).", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "message": { + "name": "message", + "title": "Message", + "description": "The text shown with every attachment of this task. Left empty, the recipient sees the file name. Like the title it is a single line, and anything beyond 4096 bytes is shortened.", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "priority": { + "name": "priority", + "title": "Priority", + "description": "How urgently every notification of this task is shown. See [message priority](https://docs.ntfy.sh/publish/#message-priority).", + "type": "string", + "parameterType": "string", + "value": "default", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "notification_tags": { + "name": "notification_tags", + "title": "Notification tags", + "description": "Tags shown with the notification, as a comma separated list. A tag that names an emoji is shown as that emoji. See [tags and emojis](https://docs.ntfy.sh/publish/#tags-emojis) and the [list of emoji names](https://docs.ntfy.sh/emojis/).", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + } + }, + "properties_advanced": { + "click": { + "name": "click", + "title": "Click URL", + "description": "The URL opened when the notification is tapped. See [click action](https://docs.ntfy.sh/publish/#click-action).", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": true, + "visibleInDialog": true, + "properties": {} + } + }, + "actions": { + "validate_connection": { + "label": "Validate connection", + "description": "Ask the server who the configured token belongs to.", + "icon": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIyNCIgaGVpZ2h0PSIyNCIgdmlld0JveD0iMCAwIDI0IDI0IiBmaWxsPSJjdXJyZW50Q29sb3IiIGZpbGwtcnVsZT0iZXZlbm9kZCIgZm9jdXNhYmxlPSJmYWxzZSIgYXJpYS1oaWRkZW49InRydWUiPgo8cGF0aCBkPSJNOS40IDE4LjIgMy4xIDExLjlsMi4zLTIuMyA0IDRMMTguNiA0bDIuMyAyLjN6Ii8+Cjwvc3ZnPgo=" + } + }, + "required": [], + "distanceMeasureRange": null, + "backendType": "python", + "is_deprecated": false, + "tags": [ + "WorkflowTask", + "PythonPlugin" + ], + "pluginType": "customtask", + "relatedPlugins": [] + }, + "cmem_plugin_ntfy-SendMessage": { + "pluginId": "cmem_plugin_ntfy-SendMessage", + "title": "Send ntfy message", + "categories": [ + "Uncategorized" + ], + "main_category": "Uncategorized", + "description": "Send a push notification to a topic on an ntfy server.", + "markdownDocumentation": "## What it does\n\nSends a push notification through ntfy, so a workflow can tell somebody what\nit did without them watching the workspace.\n\n## What flows through the ports\n\nNotifications arrive on one input port as entities carrying the paths `topic`,\n`title`, `message`, `priority`, `tags` and `click`, and one notification is\nsent per entity - where an entity's `priority` is a number from 1 to 5, one of\nntfy's names from `min` to `max`, or `urgent`, which ntfy reads as `max`. That\nport is replaced by the task's own fields when a message is configured on the\ntask itself, which then sends exactly one notification. There is no output port\neither way: the task is a terminal step and hands nothing on.\n\n## Where it sits\n\nIt is the sending half of this package. **Get ntfy messages** reads a topic in\nthe other direction, and **Send ntfy attachment** sends a file produced earlier\nin the workflow.\n\n## Caveats\n\n- **The two ways of supplying a notification exclude each other.** Either the\n task sends the one notification configured on it, or it sends one per entity\n that arrives - there is no run in which a constant message goes out a hundred\n times. Markdown, Delay, Icon URL and Email copy are the exception: they are\n never carried by an entity and apply either way.\n- **The topic is the one field an entity may leave to the task.** An entity\n that names no topic is sent to the topic configured here, which spares a\n workflow from attaching a constant to every entity.\n- **An entity that carries no message fails the run**, because a notification\n without text says nothing. So does a run that has neither: a task with no\n message configured and nothing connected to its port fails rather than\n sending silence.\n- **The first rejected request fails the run.** A rate limit is retried up to\n three times first, since the server has said it took nothing. A server error\n is not retried at all: it may have delivered the notification before failing\n to answer, and a duplicate alert is worse than a failed run. Notifications\n already sent are not taken back, and the report says how many went out\n before the failure.\n- **A message longer than 4096 bytes is shortened**, marked with an ellipsis,\n rather than refused by the server. The report counts how often that happened.\n- **An unprotected topic is public.** A topic needs no sign-up and carries no\n password of its own, so on a server without access control the only thing\n between a notification and anybody who guesses the name is the name itself -\n and then every title and message a workflow sends is readable, along with\n what ran and when. Give the topic a long random suffix rather than a word\n like `alerts`, and protect it with the server's own\n [access control](https://docs.ntfy.sh/config/#access-control) where that is available.\n- **A notification cannot be recalled.** The test action below sends a real\n notification to the configured topic.\n", + "pluginIcon": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIyNCIgaGVpZ2h0PSIyNCIgdmlld0JveD0iMCAwIDI0IDI0IiBmaWxsPSJjdXJyZW50Q29sb3IiIGZpbGwtcnVsZT0iZXZlbm9kZCIgZm9jdXNhYmxlPSJmYWxzZSIgYXJpYS1oaWRkZW49InRydWUiPgo8ZyB0cmFuc2Zvcm09InRyYW5zbGF0ZSgtMS42IDEuMSkgc2NhbGUoMC43OCkiPjxwYXRoIGQ9Ik0xMiAyMi4yYTIuMyAyLjMgMCAwIDAgMi4zLTIuMkg5LjdhMi4zIDIuMyAwIDAgMCAyLjMgMi4yeiIvPjxwYXRoIGQ9Ik0xOSAxNi40di01LjNjMC0zLjMtMS44LTYtNC44LTYuN3YtLjdhMi4yIDIuMiAwIDAgMC00LjQgMHYuN0M2LjggNS4xIDUgNy44IDUgMTEuMXY1LjNMMi44IDE4LjZ2MS4xaDE4LjR2LTEuMXoiLz48L2c+PHBhdGggZD0iTTE5LjYgNy42IDIzIDEyLjFoLTIuMXY2LjNoLTIuNnYtNi4zSDE2eiIvPgo8L3N2Zz4K", + "properties": { + "server": { + "name": "server", + "title": "Server", + "description": "The base URL of the ntfy server. The default is the public service; a self-hosted ntfy is named here instead, for example https://ntfy.example.org - see [installing ntfy](https://docs.ntfy.sh/install/).", + "type": "string", + "parameterType": "string", + "value": "https://ntfy.sh", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "access_token": { + "name": "access_token", + "title": "Access token", + "description": "The access token to authenticate with. A public topic needs none, so this can be left empty. See [access tokens](https://docs.ntfy.sh/config/#access-tokens).", + "type": "string", + "parameterType": "password", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "topic": { + "name": "topic", + "title": "Topic", + "description": "The topic the notification is sent to. An entity that names no topic of its own is sent here. A topic needs no sign-up and has no password of its own, so give it a long random suffix nobody can guess rather than a plain word - see [picking a topic](https://docs.ntfy.sh/publish/#picking-a-topic).", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "title": { + "name": "title", + "title": "Title", + "description": "The title of the notification the task sends itself. See [message titles](https://docs.ntfy.sh/publish/#message-title).", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "message": { + "name": "message", + "title": "Message", + "description": "The text of the notification. Configuring it makes the task send that one notification and removes its input port; left empty, each notification arrives on that port instead.", + "type": "string", + "parameterType": "multiline string", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "priority": { + "name": "priority", + "title": "Priority", + "description": "How urgently the notification is shown, for the one the task sends itself. See [message priority](https://docs.ntfy.sh/publish/#message-priority).", + "type": "string", + "parameterType": "string", + "value": "default", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "notification_tags": { + "name": "notification_tags", + "title": "Notification tags", + "description": "Tags shown with the notification, as a comma separated list. A tag that names an emoji is shown as that emoji. Used for the notification the task sends itself. See [tags and emojis](https://docs.ntfy.sh/publish/#tags-emojis) and the [list of emoji names](https://docs.ntfy.sh/emojis/).", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "click": { + "name": "click", + "title": "Click URL", + "description": "The URL opened when the notification is tapped. Used for the notification the task sends itself. See [click action](https://docs.ntfy.sh/publish/#click-action).", "type": "string", "parameterType": "string", - "value": null, + "value": "", "advanced": false, "visibleInDialog": true, "properties": {} + } + }, + "properties_advanced": { + "markdown": { + "name": "markdown", + "title": "Markdown", + "description": "If enabled, the message is formatted as Markdown. Note that the Android app shows the markup as typed; the web app and the desktop clients render it. See [Markdown formatting](https://docs.ntfy.sh/publish/#markdown-formatting).", + "type": "string", + "parameterType": "boolean", + "value": "false", + "advanced": true, + "visibleInDialog": true, + "properties": {} }, - "user": { - "name": "user", - "title": "User", - "description": "The user account which will receive the message. You can search for users if the connection was successful (Base URl, bot + token).", + "delay": { + "name": "delay", + "title": "Delay", + "description": "Holds the notification back: a duration such as `30m`, a timestamp, or a time of day such as `9am`, which the server reads in its own time zone. Between ten seconds and three days. See [scheduled delivery](https://docs.ntfy.sh/publish/#scheduled-delivery).", "type": "string", "parameterType": "string", "value": "", - "advanced": false, + "advanced": true, "visibleInDialog": true, "properties": {} }, - "channel": { - "name": "channel", - "title": "Channel", - "description": "The channel which will receive the message. You can search for channels if the connection was successful (Base URl, bot + token). If you want to send your message to multiple channels, separate them with a comma.", + "icon": { + "name": "icon", + "title": "Icon URL", + "description": "The URL of a PNG or JPEG shown as the notification's icon. See [icons](https://docs.ntfy.sh/publish/#icons).", "type": "string", "parameterType": "string", "value": "", - "advanced": false, + "advanced": true, "visibleInDialog": true, "properties": {} }, - "message": { - "name": "message", - "title": "Message", - "description": "The message size is limited to a configured maximum (e.g. 16383 characters).", + "email": { + "name": "email", + "title": "Email copy", + "description": "An address the server also sends the notification to as mail. Needs a server configured for mail delivery. See [e-mail notifications](https://docs.ntfy.sh/publish/#e-mail-notifications).", "type": "string", - "parameterType": "multiline string", + "parameterType": "string", "value": "", - "advanced": false, + "advanced": true, "visibleInDialog": true, "properties": {} } }, - "properties_advanced": {}, - "actions": {}, - "required": [ - "url", - "access_token", - "bot_name" - ], + "actions": { + "validate_connection": { + "label": "Validate connection", + "description": "Ask the server who the configured token belongs to.", + "icon": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIyNCIgaGVpZ2h0PSIyNCIgdmlld0JveD0iMCAwIDI0IDI0IiBmaWxsPSJjdXJyZW50Q29sb3IiIGZpbGwtcnVsZT0iZXZlbm9kZCIgZm9jdXNhYmxlPSJmYWxzZSIgYXJpYS1oaWRkZW49InRydWUiPgo8cGF0aCBkPSJNOS40IDE4LjIgMy4xIDExLjlsMi4zLTIuMyA0IDRMMTguNiA0bDIuMyAyLjN6Ii8+Cjwvc3ZnPgo=" + }, + "send_test_notification": { + "label": "Send a test notification", + "description": "Send a real notification to the configured topic.", + "icon": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIyNCIgaGVpZ2h0PSIyNCIgdmlld0JveD0iMCAwIDI0IDI0IiBmaWxsPSJjdXJyZW50Q29sb3IiIGZpbGwtcnVsZT0iZXZlbm9kZCIgZm9jdXNhYmxlPSJmYWxzZSIgYXJpYS1oaWRkZW49InRydWUiPgo8ZyB0cmFuc2Zvcm09InRyYW5zbGF0ZSgtMS42IDEuMSkgc2NhbGUoMC43OCkiPjxwYXRoIGQ9Ik0xMiAyMi4yYTIuMyAyLjMgMCAwIDAgMi4zLTIuMkg5LjdhMi4zIDIuMyAwIDAgMCAyLjMgMi4yeiIvPjxwYXRoIGQ9Ik0xOSAxNi40di01LjNjMC0zLjMtMS44LTYtNC44LTYuN3YtLjdhMi4yIDIuMiAwIDAgMC00LjQgMHYuN0M2LjggNS4xIDUgNy44IDUgMTEuMXY1LjNMMi44IDE4LjZ2MS4xaDE4LjR2LTEuMXoiLz48L2c+PHBhdGggZD0iTTE5LjYgNy42IDIzIDEyLjFoLTIuMXY2LjNoLTIuNnYtNi4zSDE2eiIvPgo8L3N2Zz4K" + } + }, + "required": [], "distanceMeasureRange": null, "backendType": "python", "is_deprecated": false, @@ -7106,7 +8671,7 @@ ], "main_category": "Uncategorized", "description": "Executes a custom Salesforce Object Query (SOQL) to return sets of data your organization's Salesforce account.", - "markdownDocumentation": "\nThis task executes a custom Salesforce Object Query (SOQL)\nand returns sets of tabular data from your organization's Salesforce account.\n\n> Use the Salesforce Object Query Language (SOQL) to search your organization's\n> Salesforce data for specific information. SOQL is similar to the SELECT statement in\n> the widely used Structured Query Language (SQL) but is designed specifically for\n> Salesforce data.\n-- [developer.salesforce.com](https://developer.salesforce.com/docs/atlas.en-us.soql_sosl.meta/soql_sosl/sforce_api_calls_soql.htm)\n\nSOQL uses the SELECT statement combined with filtering statements to return sets of\ndata, which can optionally be ordered. For a complete description of the syntax, see\n[Salesforce SOQL SELECT Syntax](https://developer.salesforce.com/docs/atlas.en-us.soql_sosl.meta/soql_sosl/sforce_api_calls_soql_select.htm).\n\nIn the Advanced Options section, you can enable / disable the validation of your\nSOQL Query. By default, this Parse SOQL option is set `True` (enabled).\n\nExamples:\n\nRetrieve all standard fields from all Lead resources. (without parser validation)\n```\nSELECT FIELDS(STANDARD) FROM Lead\n```\nRetrieve first name and last name of all Contact resources. (with parser validation)\n```\nSELECT Contact.Firstname, Contact.Lastname FROM Contact\n```\n\nPlease refer to the [Salesforce Standard Objects list](https://developer.salesforce.com/docs/atlas.en-us.238.0.object_reference.meta/object_reference/sforce_api_objects_list.htm) of the Salesforce Platform data\nmodel in order to get an overview of the available objects and fields.\n", + "markdownDocumentation": "\nThis task executes a custom Salesforce Object Query (SOQL)\nand returns sets of tabular data from your organization's Salesforce account.\n\n> Use the Salesforce Object Query Language (SOQL) to search your organization's\n> Salesforce data for specific information. SOQL is similar to the SELECT statement in\n> the widely used Structured Query Language (SQL) but is designed specifically for\n> Salesforce data.\n-- [developer.salesforce.com](https://developer.salesforce.com/docs/atlas.en-us.soql_sosl.meta/soql_sosl/sforce_api_calls_soql.htm)\n\nSOQL uses the SELECT statement combined with filtering statements to return sets of\ndata, which can optionally be ordered. For a complete description of the syntax, see\n[Salesforce SOQL SELECT Syntax](https://developer.salesforce.com/docs/atlas.en-us.soql_sosl.meta/soql_sosl/sforce_api_calls_soql_select.htm).\n\nThis task ignores any entities it receives; the query comes only from the SOQL Query\nparameter. A query that matches no records fails with an error rather than returning\nan empty result.\n\nExamples:\n\nRetrieve all standard fields from all Lead resources.\n```\nSELECT FIELDS(STANDARD) FROM Lead\n```\nRetrieve first name and last name of all Contact resources.\n```\nSELECT Contact.Firstname, Contact.Lastname FROM Contact\n```\n\nPlease refer to the [Salesforce Standard Objects list](https://developer.salesforce.com/docs/atlas.en-us.238.0.object_reference.meta/object_reference/sforce_api_objects_list.htm) of the Salesforce Platform data\nmodel in order to get an overview of the available objects and fields.\n", "pluginIcon": null, "properties": { "username": { @@ -7158,7 +8723,7 @@ "dataset": { "name": "dataset", "title": "Dataset", - "description": "In addition to have direct output of the fetched entities of your SOQL query, you can directly write the response to a JSON dataset (mostly for debugging purpose).", + "description": "Besides the direct output of the fetched entities, write the complete raw query response - including Salesforce's internal metadata for each record - to a JSON dataset (mostly for debugging purposes).", "type": "string", "parameterType": "string", "value": "", @@ -7191,14 +8756,14 @@ "Uncategorized" ], "main_category": "Uncategorized", - "description": "Executes a custom SQL query on the first input Spark dataframe and returns the result as its output.", - "markdownDocumentation": "Executes a custom SQL query on the first input Spark dataframe and returns the result as its output.\n", + "description": "Executes a Spark SQL query on the input entities and returns the result as its output.", + "markdownDocumentation": "The Spark SQL query operator runs a [Spark SQL](https://spark.apache.org/docs/latest/sql-ref.html) query against the\nentities of its single input and passes the query result to the next workflow task.\n\nThe operator requires the Spark execution backend (`execution.manager.plugin = SparkExecutionManager`). In local\nworkflow execution the task fails because no executor is available for it.\n\n## Command\n\n\nThe **Command** parameter holds the SQL query. The input entities are made available as a table named `dataset`,\nregardless of which task is connected to the input. Each column holds all values of its path as an array of strings,\nsince an entity can have several values per path. Use `try_element_at(column, 1)` to read the first value, which\nreturns NULL if the entity has no value, and cast it to compare it as a number. For example, this query keeps the\nentities whose first `age` value is above 30 and returns two of their columns:\n\n```sql\nSELECT try_element_at(name, 1) AS name, try_element_at(age, 1) AS age\nFROM dataset\nWHERE try_cast(try_element_at(age, 1) AS INT) > 30\n```\n\nThe columns hold plain strings instead of arrays if the configuration option `spark.interpreter.options.columnType`\nis set to any value other than `sequence`.\n\nThe columns of the `dataset` table are named after the paths of the input schema: the column headers of a CSV file or\nthe full property URIs of RDF data. Enclose a column name in backticks when it contains characters that are not valid\nin an SQL identifier, such as a URI:\n\n```sql\nSELECT `http://xmlns.com/foaf/0.1/name` AS name FROM dataset\n```\n\nThe entity URIs of the input are not part of the table and cannot be referenced in the query.\n\nThe table name is applied by a case-sensitive text replacement: every occurrence of the character sequence `dataset` in\nthe query is replaced with the internal name of the input table, also inside longer names such as `my_dataset` or inside\na URI, while `DATASET` or `Dataset` is left unchanged. Write the table name in lower case and make sure that column names,\naliases and string literals in the query do not contain `dataset`.\n\n## Output\n\nThe columns of the query result become the paths of the output schema, in the order the query returns them. A column\nname that is not valid in a URI, e.g. an alias containing spaces, is URL-encoded in the path, so prefer plain\nalphanumeric aliases. The output schema declares all result values as strings and keeps the type of the input entities.\nConnect a dataset or another operator to the output to use the result.\n\n", "pluginIcon": null, "properties": { "command": { "name": "command", "title": "Command", - "description": "SQL command. The name of the table in the statement must be 'dataset', regardless the input.", + "description": "SQL command. The name of the table in the statement must be 'dataset', regardless of the input.", "type": "string", "parameterType": "code-sql", "value": "", @@ -7227,7 +8792,7 @@ ], "main_category": "Uncategorized", "description": "A task that executes a SPARQL Construct query on a SPARQL enabled data source and outputs the SPARQL result. If the result should be written to the same RDF store it is read from, the SPARQL Update operator is preferable.", - "markdownDocumentation": "A task that executes a SPARQL Construct query on a SPARQL enabled data source and outputs the SPARQL result. If the result should be written to the same RDF store it is read from, the SPARQL Update operator is preferable.\n", + "markdownDocumentation": "The SPARQL Construct query operator runs a [SPARQL 1.1 CONSTRUCT query](https://www.w3.org/TR/sparql11-query/#construct)\nagainst a connected RDF dataset and passes the resulting RDF statements to the next workflow task.\nConnect an RDF dataset as its input and an RDF output dataset as its output to write the result.\nThe operator itself does not write to a dataset.\n\n## Construct query\n\n\nThe **Construct query** parameter defines which RDF statements to produce. For example, this query copies names\nfrom the source dataset into a different property in the output:\n\n```sparql\nPREFIX ex: \n\nCONSTRUCT { ?person ex:displayName ?name }\nWHERE { ?person ex:name ?name }\n```\n\nThe query must be a valid CONSTRUCT query; other query forms such as SELECT and UPDATE are rejected when the task is\nconfigured. The query is executed as written, without template variable substitution. Include any required `PREFIX`\ndeclarations in the query.\n\nThe source dataset's **Graph** setting is not automatically added to this CONSTRUCT query. To read a particular named\ngraph, specify it in the query, for example:\n\n```sparql\nCONSTRUCT { ?s ?p ?o }\nWHERE { GRAPH { ?s ?p ?o } }\n```\n\nConfigure the destination graph on the connected output dataset if the result should be written to a named graph.\n\n## Use temporary file\n\n\n**Use temporary file** is enabled by default. The operator saves the complete CONSTRUCT result to a temporary file\nbefore passing it to the output. This avoids reading from a store while the downstream task writes the result back\nto that same store. It requires enough temporary disk space for the result; the file is removed after execution.\n\nDisable this option to pass the result to the output without first saving it to a temporary file. This avoids the\nintermediate file, but reading and writing may then overlap. The option controls this operator's intermediate file,\nnot how the source endpoint processes the query internally.\n\nIf the intention is to change data within the same RDF store, consider the **SPARQL Update query** operator instead.\n", "pluginIcon": "data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiIHN0YW5kYWxvbmU9Im5vIj8+CjxzdmcKICAgd2lkdGg9IjMycHgiCiAgIGhlaWdodD0iMzJweCIKICAgdmlld0JveD0iMCAwIDMyIDMyIgogICBmaWxsPSJjdXJyZW50Q29sb3IiCiAgIHZlcnNpb249IjEuMSIKICAgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIgogICB4bWxuczpyZGY9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkvMDIvMjItcmRmLXN5bnRheC1ucyMiCiAgIHhtbG5zOmNjPSJodHRwOi8vY3JlYXRpdmVjb21tb25zLm9yZy9ucyMiCiAgIHhtbG5zOmRjPSJodHRwOi8vcHVybC5vcmcvZGMvZWxlbWVudHMvMS4xLyI+CiAgPHRpdGxlCiAgICAgaWQ9InRpdGxlMSI+IFNQQVJRTCBDb25zdHJ1Y3QgcXVlcnk8L3RpdGxlPgogIDxwYXRoCiAgICAgZD0iTSAxNiAyIEEgNCA0IDAgMCAwIDEyIDYgQSA0IDQgMCAwIDAgMTMuNjQ4NDM4IDkuMjM2MzI4MSBMIDguNDU1MDc4MSAyMC4yNzM0MzggQSA0IDQgMCAwIDAgNyAyMCBBIDQgNCAwIDAgMCAzIDI0IEEgNCA0IDAgMCAwIDcgMjggQSA0IDQgMCAwIDAgMTAuOTkyMTg4IDI0LjI1IEwgMTggMjQuMjUgTCAxOC4zODY3MTkgMjQuMjUgQyAxOC45NjAxMTYgMjYuODE5Mjk5IDIxLjI2MjI4MSAyOC43NSAyNCAyOC43NSBDIDI3LjE2Njc1NCAyOC43NSAyOS43NSAyNi4xNjY3NTQgMjkuNzUgMjMgQyAyOS43NSAxOS44MzMyNDYgMjcuMTY2NzU0IDE3LjI1IDI0IDE3LjI1IEMgMjMuMzQ0MTYgMTcuMjUgMjIuNzEzMDIyIDE3LjM2MDYyIDIyLjEyNSAxNy41NjQ0NTMgTCAxOC4zMTgzNTkgOS4yNTk3NjU2IEEgNCA0IDAgMCAwIDIwIDYgQSA0IDQgMCAwIDAgMTYgMiB6IE0gMTUuMDA1ODU5IDkuODc1IEEgNCA0IDAgMCAwIDE2IDEwIEEgNCA0IDAgMCAwIDE2Ljk1NTA3OCA5Ljg4NDc2NTYgTCAyMC43ODMyMDMgMTguMjM4MjgxIEMgMTkuMzIwNzI2IDE5LjIyOTk3NyAxOC4zMzY3MTkgMjAuODc1OTU3IDE4LjI1NTg1OSAyMi43NSBMIDE4IDIyLjc1IEwgMTAuNzk4ODI4IDIyLjc1IEEgNCA0IDAgMCAwIDkuNzMyNDIxOSAyMS4wNzgxMjUgTCAxNS4wMDU4NTkgOS44NzUgeiBNIDI0IDE4Ljc1IEMgMjYuMzU2MDk0IDE4Ljc1IDI4LjI1IDIwLjY0MzkwNiAyOC4yNSAyMyBDIDI4LjI1IDI1LjM1NjA5NCAyNi4zNTYwOTQgMjcuMjUgMjQgMjcuMjUgQyAyMS42NDM5MDYgMjcuMjUgMTkuNzUgMjUuMzU2MDk0IDE5Ljc1IDIzIEMgMTkuNzUgMjAuNjQzOTA2IDIxLjY0MzkwNiAxOC43NSAyNCAxOC43NSB6IE0gMjMgMjAgTCAyMyAyMiBMIDIxIDIyIEwgMjEgMjQgTCAyMyAyNCBMIDIzIDI2IEwgMjUgMjYgTCAyNSAyNCBMIDI3IDI0IEwgMjcgMjIgTCAyNSAyMiBMIDI1IDIwIEwgMjMgMjAgeiAiIC8+CiAgPG1ldGFkYXRhPgogICAgPHJkZjpSREY+CiAgICAgIDxjYzpXb3JrCiAgICAgICAgIHJkZjphYm91dD0iIj4KICAgICAgICA8ZGM6dGl0bGU+U1BBUlFMIENvbnN0cnVjdCBxdWVyeTwvZGM6dGl0bGU+CiAgICAgIDwvY2M6V29yaz4KICAgIDwvcmRmOlJERj4KICA8L21ldGFkYXRhPgo8L3N2Zz4=", "properties": { "query": { @@ -7244,7 +8809,7 @@ "tempFile": { "name": "tempFile", "title": "Use temporary file", - "description": "When copying directly to the same SPARQL Endpoint or when copying large amounts of triples, set to True by default", + "description": "Save the complete CONSTRUCT result to a temporary file before passing it to the output. Useful when reading from and writing to the same store. Enabled by default.", "type": "string", "parameterType": "boolean", "value": "true", @@ -7275,7 +8840,7 @@ ], "main_category": "Uncategorized", "description": "A task that executes a SPARQL Select query and outputs the SPARQL result.", - "markdownDocumentation": "The SPARQL SELECT plugin is a task for executing SPARQL SELECT queries on an RDF data source.\nIt can be used in a workflow, connecting an input to an output. A\n[SPARQL 1.1 SELECT](https://www.w3.org/TR/sparql11-query/#select) query is supported; the simplest example is\n`SELECT * WHERE { ?s ?p ?o }`.\n\n## Input and output\n\nThe _input_ depends on the configuration:\n\n- By default, the query is executed against the connected input, which must be a _SPARQL endpoint_\n (i.e. an RDF dataset).\n- When **Use fallback RDF dataset** (`useDefaultDataset`) is enabled, the query is executed against the\n fallback RDF dataset (as configured in `dataset.defaultRdf`) instead. The input port then depends on what\n the template references:\n - If the template references input entity properties (`input.entity.*`), the task accepts an entity input\n and generates one query per input entity.\n - If it references only parameters of the input task (`input.config.*`), an input connection is still\n required \u2014 it supplies the parameter values \u2014 but the query is rendered and executed only once.\n - If it references neither, the task has no input port.\n\nThe _output_ is an entity table built from the query's\n[SPARQL results](https://www.w3.org/TR/sparql11-results-json/#json-result-object): each projected variable becomes\na column, and each result binding becomes a row.\n\nThe [result size](https://www.w3.org/TR/sparql11-query/#modResultLimit) can be capped with the `limit` parameter,\nand a query timeout (in milliseconds) can be set via `sparqlTimeout`.\n\n## Automatic `FROM` clause injection\n\nIf the SPARQL source is defined on a specific graph, a `FROM` clause will be added to the query at execution time,\nexcept when there already exists a `GRAPH` or `FROM` clause in the query. `FROM NAMED` clauses are not injected.\n\n## Templating\n\nThe select query is rendered by a template engine before execution.\n[`Jinja`](https://jinja.palletsprojects.com/) is the default and is described below; for the deprecated `Simple`\nand `Velocity Engine` modes, see \"Legacy template engines\" at the end.\n\nJinja uses `{{ ... }}` for value expressions and `{% ... %}` for control flow such as conditionals.\n\n### Template variables\n\nThe following variables are available:\n\n- `input.config.`: a parameter of the task connected to the input port. `` is a parameter id\n of that task's plugin, e.g. `graph` on a SPARQL dataset.\n- `output.config.`: a parameter of the task the output is connected to.\n- `input.entity.`: the value(s) of the given property of the current input entity. Only available\n with **Use fallback RDF dataset** enabled, since only then the task receives input entities\n (see _Input and output_ above).\n- `project.`: a project-scoped template variable.\n- `global.`: a global template variable.\n\nA single-valued entity property is inserted as a plain string. A multi-valued property can be iterated with\n`{% for value in input.entity. %}`; inserting it directly concatenates all values without a\nseparator. Referencing a variable that is not available at execution time \u2014 an unknown parameter name or an\nentity property without a value \u2014 fails the query generation with an error.\n\nParameter, property and variable names must be valid Jinja identifiers (`[a-zA-Z_][a-zA-Z0-9_]*`);\nbracket-subscript access such as `input.entity[\"urn:prop:label\"]` is not supported.\n\nFor example, to query the named graph that is configured on the input dataset:\n\n```sparql\nSELECT * WHERE { GRAPH <{{ input.config.graph | validate_uri }}> { ?s ?p ?o } }\n```\n\n### Default scope\n\nThe `defaultScope` parameter declares one scope whose variables are additionally exposed at the top level of the\ntemplate context, so they can be referenced without the scope prefix. It defaults to `input.entity`, which means\na template may write `{{ property }}` as a shorthand for `{{ input.entity.property }}`:\n\n```\n{{ property }} \u2261 {{ input.entity.property }}\n```\n\nBoth forms resolve to the same value. Set `defaultScope` to the empty string to disable this aliasing and require\nevery variable to be addressed with its full scope.\n\n### Filters\n\nValues are inserted verbatim by default, so URI brackets (`<...>`) and quotation marks around literals must be\nwritten in the template. The following filters are provided to render values safely:\n\n- `validate_uri`: validates that the value is a valid absolute IRI and returns it unchanged. Throws a validation\n error otherwise. Wrap the output in `<...>` in the template.\n- `escape_literal`: escapes backslashes, quotes, newlines, carriage returns and tabs so the value can be used\n inside a short-form SPARQL string literal (`\"...\"` or `'...'`). No enclosing quotes are added.\n- `escape_multiline_literal`: escapes backslashes and breaks any run of three or more consecutive single or double\n quotes. Use for values that are wrapped in triple-quoted SPARQL literals (`\"\"\"...\"\"\"` or `'''...'''`).\n\nAll transformer plugins are also available as Jinja filters under their plugin id (for example `lowerCase`,\n`trim`, `urlEncode`).\n\n### Input schema inference\n\nThe input schema (the entity properties the task expects) is derived by scanning the raw template for\n`input.entity.` references (or bare references resolved via `defaultScope`). This scan operates\non the template text before rendering, so SPARQL line comments (`# ...`) are **not** stripped: a\ncommented-out line such as\n\n```sparql\n# {{ input.entity.property }}\n```\n\nwill still cause `property` to appear in the inferred input schema.\n\n### Output schema inference\n\nThe output schema is derived from the raw template by a heuristic, without rendering it. The heuristic takes\nthe projection between `SELECT` and the first `WHERE`, `FROM` or `{`, drops a leading `DISTINCT` / `REDUCED`,\nand then:\n\n- For `SELECT *`, collects every distinct `?var` token in the query.\n- Otherwise, collects each top-level `?var` and the trailing `AS ?alias` from parenthesised expressions\n (e.g. `(COUNT(?s) AS ?count)` yields `count`).\n\nEach variable becomes a string-typed path. If no variables can be detected (e.g. the projection is produced by\na Jinja expression), the output port is reported with an unknown schema.\n\n### Validation\n\nAt task creation, the Jinja template is checked against the available template variables:\n\n- Every `project.<...>` or `global.<...>` reference must resolve to a known variable, matched on the full\n scoped name (so e.g. `project.metaData.label` is looked up at that exact scope).\n- Every `input.<...>` or `output.<...>` reference must use `config` or `entity` as its second segment.\n\nBare references are resolved through `defaultScope` before applying the same rules. The template is not\nrendered and the resulting SPARQL is not parsed.\n\n### Legacy template engines\n\nIn addition to Jinja, two deprecated template engines are supported for backwards compatibility: `Simple`\nand [`Velocity Engine`](https://velocity.apache.org/engine/2.4.1/user-guide.html). Their syntax is identical\nto the one used by the `SPARQL Update operator` and is documented there.\n", + "markdownDocumentation": "The SPARQL SELECT plugin is a task for executing SPARQL SELECT queries on an RDF data source.\nIt can be used in a workflow, connecting an input to an output. A\n[SPARQL 1.1 SELECT](https://www.w3.org/TR/sparql11-query/#select) query is supported; the simplest example is\n`SELECT * WHERE { ?s ?p ?o }`.\n\n## Input and output\n\nThe _input_ depends on the configuration:\n\n- By default, the query is executed against the connected input, which must be a _SPARQL endpoint_\n (i.e. an RDF dataset).\n- When **Use fallback RDF dataset** (`useDefaultDataset`) is enabled, the query is executed against the\n fallback RDF dataset (as configured in `dataset.defaultRdf`) instead. The input port then depends on what\n the template references:\n - If the template references input entity properties (`input.entity.*`), the task accepts an entity input\n and generates one query per input entity.\n - If it references only parameters of the input task (`input.config.*`), an input connection is still\n required \u2014 it supplies the parameter values \u2014 but the query is rendered and executed only once.\n - If it references neither, the task has no input port.\n\nThe _output_ is an entity table built from the query's\n[SPARQL results](https://www.w3.org/TR/sparql11-results-json/#json-result-object): each projected variable becomes\na column, and each result binding becomes a row.\n\nThe [result size](https://www.w3.org/TR/sparql11-query/#modResultLimit) can be capped with the `limit` parameter,\nand a query timeout (in milliseconds) can be set via `sparqlTimeout`.\n\n## Automatic `FROM` clause injection\n\nIf the SPARQL source is defined on a specific graph, a `FROM` clause will be added to the query at execution time,\nexcept when there already exists a `GRAPH` or `FROM` clause in the query. `FROM NAMED` clauses are not injected.\n\n## Templating\n\nThe select query is rendered by a template engine before execution.\n[`Jinja`](https://jinja.palletsprojects.com/) is the default and is described below; for the deprecated `Simple`\nand `Velocity Engine` modes, see \"Legacy template engines\" at the end.\n\nJinja uses `{{ ... }}` for value expressions and `{% ... %}` for control flow such as conditionals.\n\n### Template variables\n\nThe following variables are available:\n\n- `input.config.`: a parameter of the task connected to the input port. `` is a parameter id\n of that task's plugin, e.g. `graph` on a SPARQL dataset.\n- `output.config.`: a parameter of the task the output is connected to.\n- `input.entity.`: the value(s) of the given property of the current input entity. Only available\n with **Use fallback RDF dataset** enabled, since only then the task receives input entities\n (see _Input and output_ above).\n- `project.`: a project-scoped template variable.\n- `global.`: a global template variable.\n\nA single-valued entity property is inserted as a plain string. A multi-valued property can be iterated with\n`{% for value in input.entity. %}`; inserting it directly concatenates all values without a\nseparator. Referencing a variable that is not available at execution time \u2014 an unknown parameter name or an\nentity property without a value \u2014 fails the query generation with an error.\n\nParameter, property and variable names must be valid Jinja identifiers (`[a-zA-Z_][a-zA-Z0-9_]*`);\nbracket-subscript access such as `input.entity[\"urn:prop:label\"]` is not supported.\n\nFor example, to query the named graph that is configured on the input dataset:\n\n```sparql\nSELECT * WHERE { GRAPH <{{ input.config.graph | validate_uri }}> { ?s ?p ?o } }\n```\n\n### Default scope\n\nThe `defaultScope` parameter declares one scope whose variables are additionally exposed at the top level of the\ntemplate context, so they can be referenced without the scope prefix. It defaults to `input.entity`, which means\na template may write `{{ property }}` as a shorthand for `{{ input.entity.property }}`:\n\n```\n{{ property }} \u2261 {{ input.entity.property }}\n```\n\nBoth forms resolve to the same value. Set `defaultScope` to the empty string to disable this aliasing and require\nevery variable to be addressed with its full scope.\n\n### Filters\n\nValues are inserted verbatim by default, so URI brackets (`<...>`) and quotation marks around literals must be\nwritten in the template. The following filters are provided to render values safely:\n\n- `validate_uri`: validates that the value is a valid absolute IRI and returns it unchanged. Throws a validation\n error otherwise. Wrap the output in `<...>` in the template.\n- `escape_literal`: escapes backslashes, quotes, newlines, carriage returns and tabs so the value can be used\n inside a short-form SPARQL string literal (`\"...\"` or `'...'`). No enclosing quotes are added.\n- `escape_multiline_literal`: escapes backslashes and breaks any run of three or more consecutive single or double\n quotes. Use for values that are wrapped in triple-quoted SPARQL literals (`\"\"\"...\"\"\"` or `'''...'''`).\n\nAll transformer plugins are also available as Jinja filters under their plugin id (for example `lowerCase`,\n`trim`, `urlEncode`).\n\n### Input schema inference\n\nThe input schema (the entity properties the task expects) is derived by scanning the raw template for\n`input.entity.` references (or bare references resolved via `defaultScope`). This scan operates\non the template text before rendering, so SPARQL line comments (`# ...`) are **not** stripped: a\ncommented-out line such as\n\n```sparql\n# {{ input.entity.property }}\n```\n\nwill still cause `property` to appear in the inferred input schema.\n\n### Output schema inference\n\nThe output schema is derived from the raw template by a heuristic, without rendering it. The heuristic first\nblanks out line comments, Jinja comments, string literals, IRIs and Jinja tags, so that nothing inside them is\nmistaken for query structure. It then takes the projection between `SELECT` and the first `WHERE`, `FROM` or `{`\noutside of parentheses (a keyword inside a name, e.g. `?from` or `ex:where`, is not a boundary), drops a leading\n`DISTINCT` / `REDUCED`, and then:\n\n- For `SELECT *`, collects every distinct `?var` token in the rest of the query, unless a sub-select, `EXISTS` or `MINUS`\n could bind variables that are not part of the result.\n- Otherwise, collects each top-level `?var` and the trailing `AS ?alias` from parenthesised expressions\n (e.g. `(COUNT(?s) AS ?count)` yields `count`).\n\nEach variable becomes a string-typed path. If no variables can be detected, or a Jinja tag could change the set\nof result variables (any block tag before the query, a tag inside the projection, or, for `SELECT *`, a tag anywhere after\nthe projection that is not inside an IRI or string literal), the output port is reported with an unknown schema and the\nschema is taken from the actual query results at execution time. The heuristic need not be complete but must not be\nwrong, so whenever it is in doubt (for example on unbalanced parentheses) it reports an unknown schema as well.\n\n### Validation\n\nAt task creation, the Jinja template is checked against the available template variables:\n\n- Every `project.<...>` or `global.<...>` reference must resolve to a known variable, matched on the full\n scoped name (so e.g. `project.metaData.label` is looked up at that exact scope).\n- Every `input.<...>` or `output.<...>` reference must use `config` or `entity` as its second segment.\n\nBare references are resolved through `defaultScope` before applying the same rules. The template is not\nrendered and the resulting SPARQL is not parsed.\n\n### Legacy template engines\n\nIn addition to Jinja, two deprecated template engines are supported for backwards compatibility: `Simple`\nand [`Velocity Engine`](https://velocity.apache.org/engine/2.4.1/user-guide.html). Their syntax is identical\nto the one used by the `SPARQL Update operator` and is documented there.\n", "pluginIcon": "data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiIHN0YW5kYWxvbmU9Im5vIj8+CjxzdmcKICAgICAgICB3aWR0aD0iMzJweCIKICAgICAgICBoZWlnaHQ9IjMycHgiCiAgICAgICAgdmlld0JveD0iMCAwIDMyIDMyIgogICAgICAgIHZlcnNpb249IjEuMSIKICAgICAgICBmaWxsPSJjdXJyZW50Q29sb3IiCiAgICAgICAgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIgogICAgICAgIHhtbG5zOnJkZj0iaHR0cDovL3d3dy53My5vcmcvMTk5OS8wMi8yMi1yZGYtc3ludGF4LW5zIyIKICAgICAgICB4bWxuczpjYz0iaHR0cDovL2NyZWF0aXZlY29tbW9ucy5vcmcvbnMjIgogICAgICAgIHhtbG5zOmRjPSJodHRwOi8vcHVybC5vcmcvZGMvZWxlbWVudHMvMS4xLyI+CiAgICA8dGl0bGU+U1BBUlFMIFNlbGVjdCBxdWVyeTwvdGl0bGU+CiAgICA8cGF0aAogICAgICAgICAgICBkPSJtIDI3LDcgYSA0LDQgMCAwIDEgLTQsNCA0LDQgMCAwIDEgLTQsLTQgNCw0IDAgMCAxIDQsLTQgNCw0IDAgMCAxIDQsNCB6IiAvPgogICAgPHBhdGgKICAgICAgICAgICAgZD0iTSAxMiw2IEEgNCw0IDAgMCAxIDgsMTAgNCw0IDAgMCAxIDQsNiA0LDQgMCAwIDEgOCwyIDQsNCAwIDAgMSAxMiw2IFoiIC8+CiAgICA8cGF0aAogICAgICAgICAgICBkPSJtIDEyLDI2IGEgNCw0IDAgMCAxIC00LDQgNCw0IDAgMCAxIC00LC00IDQsNCAwIDAgMSA0LC00IDQsNCAwIDAgMSA0LDQgeiIgLz4KICAgIDxwYXRoCiAgICAgICAgICAgIGQ9Im0gMTEsMTYgYSAzLDMgMCAwIDEgLTMsMyAzLDMgMCAwIDEgLTMsLTMgMywzIDAgMCAxIDMsLTMgMywzIDAgMCAxIDMsMyB6IiAvPgogICAgPHBhdGgKICAgICAgICAgICAgZD0ibSAyNy42MDcxMjMsMjIuODIwMTQ1IGMgMC42NjYwOSwtMi4wNDQ0NDIgLTAuMTUyODk2LC00LjI3OTIyIC0xLjk4MjMwMSwtNS40MDkxMyAtMS44Mjk0MDQsLTEuMTI5OTExIC00LjE5NDM1OSwtMC44NjE2NTYgLTUuNzI0MjE0LDAuNjQ5MjkxIC0xLjUyOTg1NSwxLjUxMDk0OCAtMS44Mjc0OTgsMy44NzIzODQgLTAuNzIwNDIzLDUuNzE1Njk4IDEuMTA3MDc0LDEuODQzMzE0IDMuMzMxNDk1LDIuNjkwMDI3IDUuMzg0MDYyLDIuMDQ5NDEyIGwgMS45MDUxMzgsMy43MzkwNDIgMS4zODk5MDUsLTAuNzA4MTkyIC0xLjkwNTEzNywtMy43MzkwNDMgYyAwLjc3NTc1NiwtMC41NzU4NjEgMS4zNTMzODIsLTEuMzc4NTY4IDEuNjUyOTcsLTIuMjk3MDc4IHogbSAtNS4zOTI3NjIsMS41MTQ4MjkgYyAtMS42MzIwMzksLTAuNTMwMjgyIC0yLjUyNTE5LC0yLjI4MzE4OSAtMS45OTQ5MDksLTMuOTE1MjI4IDAuNTMwMjgyLC0xLjYzMjA0IDIuMjgzMTg5LC0yLjUyNTE5MSAzLjkxNTIyOSwtMS45OTQ5MDkgMS42MzIwMzksMC41MzAyODIgMi41MjUxOSwyLjI4MzE4OSAxLjk5NDkwOCwzLjkxNTIyOCAtMC4yNTQ2NSwwLjc4MzczMyAtMC44MTAyMDcsMS40MzQyMDcgLTEuNTQ0NDU1LDEuODA4MzI1IC0wLjczNDI0OCwwLjM3NDExOCAtMS41ODcwNCwwLjQ0MTIzNCAtMi4zNzA3NzMsMC4xODY1ODQgeiIgLz4KICAgIDxwYXRoCiAgICAgICAgICAgIGQ9Ik0gNy4yNSw1LjI1IFYgNiAyNi41IDI3LjI1IGggMS41IFYgMjYuNSA2IDUuMjUgWiIgLz4KICAgIDxwYXRoCiAgICAgICAgICAgIGQ9Ik0gMjMuMjQ0MTQxLDUuOTY4NzUgMjIuNjA1NDY5LDYuMzYxMzI4MSA3LjYwNTQ2ODcsMTUuNjExMzI4IDYuOTY4NzUsMTYuMDA1ODU5IDcuNzU1ODU5NCwxNy4yODEyNSA4LjM5NDUzMTMsMTYuODg4NjcyIDIzLjM5NDUzMSw3LjYzODY3MTkgMjQuMDMxMjUsNy4yNDQxNDA2IFoiIC8+CiAgICA8cGF0aAogICAgICAgICAgICBkPSJtIDcuNTcwMzEyNSwxNS4yNzkyOTcgLTAuNTQxMDE1NiwxLjQwMDM5IDAuNzAxMTcxOCwwLjI2OTUzMiAxMS4wMDAwMDAzLDQuMjUgMC42OTkyMTgsMC4yNzE0ODQgMC41NDEwMTYsLTEuNDAwMzkxIC0wLjcwMTE3MiwtMC4yNjk1MzEgLTEwLjk5OTk5OTcsLTQuMjUgeiIgLz4KICAgIDxtZXRhZGF0YT4KICAgICAgICA8cmRmOlJERj4KICAgICAgICAgICAgPGNjOldvcmsKICAgICAgICAgICAgICAgICAgICByZGY6YWJvdXQ9IiI+CiAgICAgICAgICAgICAgICA8ZGM6dGl0bGU+U1BBUlFMIFNlbGVjdCBxdWVyeTwvZGM6dGl0bGU+CiAgICAgICAgICAgIDwvY2M6V29yaz4KICAgICAgICA8L3JkZjpSREY+CiAgICA8L21ldGFkYXRhPgo8L3N2Zz4=", "properties": { "selectQuery": { @@ -7683,7 +9248,7 @@ ], "main_category": "Uncategorized", "description": "Store embeddings into Postgres Vector Store (PGVector).", - "markdownDocumentation": "\nThis plugin workflow store embeddings into Postgres Vector Store.\n\nThe vector embeddings and its respective metadata are going to be stored into a collection inside\nthe Postgres Vector Store.\nIt is possible to specify either the name of the attributes containing the vectors as well as the\nmetadata.\n", + "markdownDocumentation": "\nThis workflow task stores embeddings and their metadata into a collection in Postgres Vector\nStore (PGVector).\n\nThe task accepts entities of any schema on its input port. Two of the incoming paths supply the\ntext and the embedding to store; every other path, or a fixed subset of them, is stored alongside\nas metadata. The task produces no output port: it is a terminal step and hands nothing on to\nlater tasks in the workflow.\n\nUse together with **Search Vector Embeddings**, which reads back the collection this task\npopulates.\n\nIf the incoming entities have no path configured for the text or the embedding, the task fails\nbefore storing anything.\n", "pluginIcon": "data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiIHN0YW5kYWxvbmU9Im5vIj8+CjxzdmcKICAgdmlld0JveD0iMCAwIDI0IDI0IgogICB2ZXJzaW9uPSIxLjEiCiAgIGlkPSJzdmc5MTkiCiAgIHNvZGlwb2RpOmRvY25hbWU9InBvc3RncmVzcWwuc3ZnIgogICBpbmtzY2FwZTp2ZXJzaW9uPSIxLjEuMiAoYjhlMjViZTgsIDIwMjItMDItMDUpIgogICB4bWxuczppbmtzY2FwZT0iaHR0cDovL3d3dy5pbmtzY2FwZS5vcmcvbmFtZXNwYWNlcy9pbmtzY2FwZSIKICAgeG1sbnM6c29kaXBvZGk9Imh0dHA6Ly9zb2RpcG9kaS5zb3VyY2Vmb3JnZS5uZXQvRFREL3NvZGlwb2RpLTAuZHRkIgogICB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciCiAgIHhtbG5zOnN2Zz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPgogIDxkZWZzCiAgICAgaWQ9ImRlZnM5MjMiIC8+CiAgPHNvZGlwb2RpOm5hbWVkdmlldwogICAgIGlkPSJuYW1lZHZpZXc5MjEiCiAgICAgcGFnZWNvbG9yPSIjZmZmZmZmIgogICAgIGJvcmRlcmNvbG9yPSIjNjY2NjY2IgogICAgIGJvcmRlcm9wYWNpdHk9IjEuMCIKICAgICBpbmtzY2FwZTpwYWdlc2hhZG93PSIyIgogICAgIGlua3NjYXBlOnBhZ2VvcGFjaXR5PSIwLjAiCiAgICAgaW5rc2NhcGU6cGFnZWNoZWNrZXJib2FyZD0iMCIKICAgICBzaG93Z3JpZD0iZmFsc2UiCiAgICAgaW5rc2NhcGU6em9vbT0iMTcuMTkyNjgxIgogICAgIGlua3NjYXBlOmN4PSIxMi4zNTk5MSIKICAgICBpbmtzY2FwZTpjeT0iMTIuODgzMzg5IgogICAgIGlua3NjYXBlOndpbmRvdy13aWR0aD0iMTkyMCIKICAgICBpbmtzY2FwZTp3aW5kb3ctaGVpZ2h0PSIxMDI3IgogICAgIGlua3NjYXBlOndpbmRvdy14PSIxNzI4IgogICAgIGlua3NjYXBlOndpbmRvdy15PSIyNSIKICAgICBpbmtzY2FwZTp3aW5kb3ctbWF4aW1pemVkPSIwIgogICAgIGlua3NjYXBlOmN1cnJlbnQtbGF5ZXI9InN2ZzkxOSIgLz4KICA8cGF0aAogICAgIGQ9Im0gMjEuMjk2MzQyLDE0LjExNTkxNSBhIDAuNDAyMDkyODQsMC4zODg1MTYyNSAwIDAgMCAtMC4wNDU4NCwtMC4wOTMyNCBRIDIxLjA4MTYyMywxMy43MTU3NDcgMjAuNDQwNjg3LDEzLjg0MzE4IGMgLTEuMzMwMTI0LDAuMjY0MTkxIC0xLjg0NDgwMiwwLjEwMTAxNSAtMi4wMzEzNzMsLTAuMDE1NTQgMS4wNzkyMTcsLTEuNTkxMzYzIDEuOTY2MjM0LC0zLjUxMzc0MSAyLjQ0NTUyOCwtNS4zMDcxMzIxIDAuMjE4NzM5LC0wLjgxNTg4NDIgMC42NDE3NCwtMi43Mzc0ODU1IDAuMDk4MTEsLTMuNjc1MzYzNyBBIDEuMjg2Njk3MSwxLjI0MzI1MiAwIDAgMCAyMC44MzIzMjcsNC42NjE3NjA1IEMgMTkuNzk0OTI4LDMuMzgyNzY1IDE4LjI3MjYwNCwyLjY5NTA5MTMgMTYuNDMxMDE5LDIuNjc2NDQyNSAxNS4yMjg3NjEsMi42NjQwMSAxNC4yMDM0MjQsMi45NDUyOTU3IDEzLjkyNTE3NiwzLjA0ODY0MTEgYSA4LjA0MTg1NjgsNy43NzAzMjQ5IDAgMCAwIC0wLjQxNDk2LC0wLjA2MzcxNyA2LjQzMzQ4NTQsNi4yMTYyNTk5IDAgMCAwIC0xLjA1NTA5MSwtMC4wOTg2ODMgQyAxMS41MDQ1NzcsMi44NzE0Nzc2IDEwLjY4MzUwNCwzLjA5MTM3NzggMTAuMDAyMzU4LDMuNTM4OTQ4NiA5LjMxMzk3NTUsMy4yODk1MjExIDYuMTUyNzIxNiwyLjI2MDczMDEgNC4xOTYxMzc5LDMuNjAwMzM0MSAzLjEwMTY0MTIsNC4zNDg2MTY0IDIuNTk4MjIwOSw1LjY4NTExMjMgMi42OTU1Mjc0LDcuNTc0MDc4MyAyLjcyODQ5OSw4LjIwOTY5MDkgMy4xMDMyNDk1LDEwLjE2NDcwNSAzLjY5NTEzMDIsMTIuMDM3MzUzIHEgMC41NTQ4ODgxLDEuNzU2MDkzIDEuMTUyMzk4MSwyLjc4MzMzIDAuNjY3NDc0MSwxLjE2MDExIDEuMzc4Mzc0MiwxLjM5MDg4OCBjIDAuMzYwMjc1MiwwLjExNTAwMSAwLjkxMTE0MjQsMC4xMTExMTYgMS40OTQxNzcsLTAuNTY2NDU2IGEgNDUuMDM0Mzk4LDQzLjUxMzgyIDAgMCAxIDEuNTY0MTQxMiwtMS43MTQxMzQgYyAwLjM0OTgyMDcsMC4xODI2MDMgMC43Mjg1OTIzLDAuMjgxMjg2IDEuMTE3ODE4MywwLjI5Mjk0MSB2IDAuMDAzMSBhIDguODQ2MDQyNSw4LjU0NzM1NzQgMCAwIDAgLTAuMTk4NjM0LDAuMjM2OTk1IGMgLTAuMjcyNjE5LDAuMzM0MTI0IC0wLjMyOTcxNjIsMC40MDQwNTcgLTEuMjA2Mjc4NiwwLjU3ODg5IC0wLjI0OTI5NzYsMC4wNDk3MyAtMC45MTE5NDY2LDAuMTgxMDQ4IC0wLjkyMTU5NjgsMC42MzAxNzMgYSAwLjQ4MjUxMTQxLDAuNDY2MjE5NSAwIDAgMCAwLjA3MzE4MSwwLjI1NDA5IGMgMC4xODI1NTAxLDAuMzI4Njg0IDAuNzQxNDU5MiwwLjQ3Mzk4OSAwLjgxNjI0ODUsMC40OTE4NjEgMS4wNzM1ODc5LDAuMjU4NzUyIDIuMDE0NDg0OSwwLjA3MTQ5IDIuNzExNzEzOSwtMC41Mjc2MDUgLTAuMDEzNjcsMS43MzM1NiAwLjA2MTkyLDMuNDMyOTMgMC4yNzc0NDQsMy45NTM1NDEgMC4xNzc3MjUsMC40Mjk2OTkgMC42MTI3OSwxLjQ3OTQ3IDEuOTg2MzM5LDEuNDc5NDcgcSAwLjMwMTU2OSw3Ljc3ZS00IDAuNjY2NjcsLTAuMDczMDQgYyAxLjQzMzA1OSwtMC4yOTY4MjYgMi4wNTU0OTgsLTAuOTA5MTI4IDIuMjk1OTUsLTIuMjU4MDU2IDAuMTIwNjI4LC0wLjY3NjAxOCAwLjMyMzI4MiwtMi4yMzM5NjkgMC40MzM0NTYsLTMuMTg2NjEgMC4wMTM2NywtMC4wNTQzOSAwLjAyODk1LC0wLjA5MzI0IDAuMDQ1ODQsLTAuMTA1Njc3IDAsMCAwLjA1NjI5LC0wLjAzNzMgMC4zNDMzODgsMC4wMjMzMSBsIDAuMDM1MzgsMC4wMDU0IDAuMjA0MjYzLDAuMDE3MDkgMC4wMTIwNiw3Ljc3ZS00IGMgMC42ODExNDUsMC4wMzAzIDEuNTM2Nzk5LC0wLjExMDMzOSAyLjAzNTM5NCwtMC4zMzQxMjQgMC41MTc4OTUsLTAuMjMzMTEgMS40NTIzNTksLTAuODAyNjc0IDEuMjgyNjc2LC0xLjI5NzY0NCBNIDQuMjU1NjQ3NiwxMS45MDM3MDMgQyAzLjY1NzMzMzUsMTAuMDExNjI5IDMuMzA4MzE2OSw4LjEwNzg5OTYgMy4yODA5NzQ2LDcuNTc0ODU1MyAzLjE5MzMxODMsNS44ODcxNDA3IDMuNjE2MzIsNC43MTMwNDQ3IDQuNTM3MTEyNiw0LjA4MzY0ODMgNi4wMTQ0MDE3LDMuMDc0MjgzMSA4LjQyOTM3MTMsMy42NjQwNTA4IDkuNDQ5MDc4NywzLjk4MjYzNDEgbCAtMC4wMDgwNCwwLjAwNzc3IEMgNy44MTQxNjkyLDUuNTc3MTA0OCA3Ljg1Mjc3MDIsOC4yOTA1MDIyIDcuODU4Mzk5NSw4LjQ1Njc4NzIgYyAwLDAuMDYzNzE3IDAuMDA0ODMsMC4xNTQ2Mjk1IDAuMDEyODY3LDAuMjc5NzMxNyAwLjAyNzM0MiwwLjQ1NTM0MSAwLjA4MDQxOSwxLjMwNTQxNDEgLTAuMDU5NTEsMi4yNjczODExIC0wLjEyODY2OTcsMC44OTM1ODcgMC4xNTYwMTIsMS43Njg1MjYgMC43ODI0NzI3LDIuNDAwMjUzIHEgMC4wOTY1MDIsMC4wOTc5MSAwLjIwMjY1NDcsMC4xODQxNTcgYyAtMC4yNzkwNTI0LDAuMjg4Mjc5IC0wLjg4NDYwNDIsMC45MjcgLTEuNTMwMzY1MywxLjY3NjgzNiAtMC40NTY3Nzc1LDAuNTI5OTM2IC0wLjc3MjAxODMsMC40MjgxNDUgLTAuODc0OTU0LDAuMzk0NzMyIEMgNi4wNzYzMjQsMTUuNTU4ODY0IDUuNzM3NzYxOCwxNS4yMDM3NiA1LjM5NTE3ODcsMTQuNjMyNjQxIDUuMDA5MTY5NiwxMy45ODA3MTEgNC42MjA3NDc5LDEzLjA1MzcxMSA0LjI1NzI1NiwxMS45MDM3MDMgbSA0LjgzMDc0MzQsMy45NTM1NDIgQSAxLjI4NjY5NzEsMS4yNDMyNTIgMCAwIDEgOC43NDA1OTExLDE1LjcxODkzMyBjIDAuMDcxNTczLC0wLjAzMDMgMC4xOTA1OTIxLC0wLjA2OTkzIDAuMzg4NDIxNywtMC4xMDg3ODUgMS4wMzI1NzQyLC0wLjIwNTkxMyAxLjE5MTgwMzIsLTAuMzUwNDQxIDEuNTQwMDE1MiwtMC43NzcwMzIgMC4wNzk2MSwtMC4wOTg2OCAwLjE2OTY4NCwtMC4yMDk3OTkgMC4yOTUxMzcsLTAuMzQ0MjI2IGEgMC4zMjE2NzQyNywwLjMxMDgxMyAwIDAgMCAwLjA1OTUxLC0wLjEwMTAxNCBjIDAuMTM2NzEyLC0wLjExNzMzMiAwLjIxODczOSwtMC4wODU0NyAwLjM1MDYyNSwtMC4wMzI2NCAwLjEyNTQ1MywwLjA1MDUxIDAuMjQ3Njg5LDAuMjAyMDI4IDAuMjk3NTQ5LDAuMzY5MDkgMC4wMjQxMywwLjA3OTI2IDAuMDQ5ODYsMC4yMjkyMjUgLTAuMDM2MTksMC4zNDU3OCAtMC43MjY5ODQsMC45ODM3MjMgLTEuNzg2OTAxLDAuOTcxMjkgLTIuNTQ3NjYwNiwwLjc4NzEzNCBtIDEuNjgzOTY0NiwtMy4wOTg4MDYgLTAuMDQxODIsMC4xMDg3ODUgYyAtMC4xMDY5NTcsMC4yNzc0IC0wLjIwNjY3NiwwLjUzNTM3NSAtMC4yNjg1OTgsMC43ODAxNCBDIDkuOTI1MTU2NywxMy42NDU4MSA5LjQwMjQzNiwxMy40MjM1NzkgOS4wMDU5NzI0LDEzLjAyMzQwNyA4LjUwMDk0MzgsMTIuNTE0NDUxIDguMjcxNzUwOSwxMS44MDY1NzQgOC4zNzYyOTUsMTEuMDgwODI2IDguNTIzNDYxLDEwLjA2NDQ2NyA4LjQ2OTU4MDYsOS4xNzk0Mjc0IDguNDQwNjI5OSw4LjcwMzg4MzUgTCA4LjQzMDE3NTUsOC41MzI5MzY0IEMgOC42NjgyMTQ0LDguMzI5MzUzOSA5Ljc2OTk0ODgsNy43NTkwMTIgMTAuNTU1NjM4LDcuOTMzMDY3MyBjIDAuMzU4NjY3LDAuMDc5MjU3IDAuNTc3NDA2LDAuMzE1NDc1MiAwLjY2NzQ3NCwwLjcyMTA4NjIgMC40NzA0NDksMi4xMDEwOTU1IDAuMDYyNzMsMi45NzYwMzQ1IC0wLjI2NTM4MSwzLjY4MDAyNTUgYSA3LjIzNzY3MTEsNi45OTMyOTI0IDAgMCAwIC0wLjE4NDk2MywwLjQyNDI2IG0gNS45MjIwMjQsMy41NTI1OTMgcSAtMC4wMTkzLDAuMjA2NjkgLTAuMDQ5ODYsMC40NjMxMTEgbCAtMC4xMTc0MTEsMC4zNDAzNCBhIDAuMzIxNjc0MjcsMC4zMTA4MTMgMCAwIDAgLTAuMDE0NDgsMC4wODM5MiBjIC0wLjAwNDgsMC4zNjkwOSAtMC4wNDM0MywwLjUwNDI5NCAtMC4wOTI0OCwwLjY3NjAxOCAtMC4wNTE0NywwLjE3Nzk0IC0wLjEwODU2NSwwLjM3OTE5MiAtMC4xNDQ3NTMsMC44MjEzMjMgLTAuMDg4NDYsMS4wOTg3MjQgLTAuNzA2MDc1LDEuNzMwNDUyIC0xLjk0MzcxNywxLjk4NjA5NSAtMS4yMTgzNDEsMC4yNTI1MzYgLTEuNDM0NjY3LC0wLjM4NTQwOCAtMS42MjQ0NTUsLTAuOTQ4NzU2IGEgNS42MjkyOTk4LDUuNDM5MjI3NCAwIDAgMCAtMC4wNjI3MywtMC4xNzYzODcgYyAtMC4xNzI5LC0wLjQ1NTM0MSAtMC4xNTI3OTYsLTEuMDk3MTcgLTAuMTI2MjU4LC0xLjk4NTMxOCAwLjAxMjg3LC0wLjQzNTkxNSAtMC4wMjAxLC0xLjQ3NzEzOCAtMC4yNjUzODEsLTIuMDU2MDI4IHEgMC4wMDQ4LC0wLjM0MTg5NCAwLjAxNTI4LC0wLjY5MzExMyBhIDAuMzIxNjc0MjcsMC4zMTA4MTMgMCAwIDAgLTAuMDEyODcsLTAuMDg3OCAxLjYwODM3MTQsMS41NTQwNjUgMCAwIDAgLTAuMDM1MzgsLTAuMTYxNjIzIGMgLTAuMDk4MTEsLTAuMzMyNTcgLTAuMzM3NzU4LC0wLjYxMDc0OCAtMC42MjcyNjUsLTAuNzI2NTI1IC0wLjExNDE5NSwtMC4wNDU4NCAtMC4zMjQwODcsLTAuMTI5NzY1IC0wLjU3NjYwMSwtMC4wNjc2IDAuMDUzODgsLTAuMjE0NDYxIDAuMTQ3MTY2LC0wLjQ1NjExOCAwLjI0ODQ5MywtMC43MTg3NTUgbCAwLjA0MjYyLC0wLjExMDMzOSBjIDAuMDQ4MjUsLTAuMTI0MzI1IDAuMTA3NzYxLC0wLjI1MjUzNSAwLjE3MTI5MSwtMC4zODg1MTYgMC4zNDI1ODMsLTAuNzM2NjI3IDAuODEyMjI4LC0xLjc0NTIxNSAwLjMwMjM3NCwtNC4wMjM0NzQ0IEMgMTEuNTkwNjI1LDcuNjg0NDE2OSAxMC45NTI5MDYsNy4yNjc5Mjc1IDkuOTg2Mjc0OCw3LjM2NDI3OTUgOS40MDcyNjExLDcuNDIyNTU3IDguODc2NDk4NSw3LjY0ODY3MzQgOC42MTE5MjE0LDcuNzc3NjYwOCBBIDQuODI1MTE0MSw0LjY2MjE5NSAwIDAgMCA4LjQ1NDMwMSw3Ljg1ODQ3MjIgQyA4LjUyODI4NjEsNi45OTkwNzQyIDguODA3MzM4Niw1LjM5MjE3MSA5Ljg1MDM2NzQsNC4zNzU4MTI2IGEgMy4yMTY3NDI3LDMuMTA4MTMgMCAwIDEgMC4yNDM2Njg2LC0wLjIxNDQ2MSAwLjI4MTQ2NDk5LDAuMjcxOTYxMzcgMCAwIDAgMC4xMTY2MDcsLTAuMDQ5NzMgQyAxMC44MTUzOSwzLjY2ODcxMyAxMS41NzM3MzcsMy40NTExNDM5IDEyLjQ2Mzk3MSwzLjQ2NDM1MzQgcSAwLjQ5NTM3OCwwLjAwNzc3IDAuOTQ0MTE0LDAuMDYyOTQgYyAxLjU2MDEyLDAuMjc1ODQ2NSAyLjYwODc3OCwxLjEyNDM2NiAzLjI0NTY5MywxLjg1MTY2ODQgMC42NTQ2MDcsMC43NDc1MDUzIDEuMDA5MjUzLDEuNTAwNDQ5NyAxLjE1MDc5LDEuOTA2ODM3NyAtMS4wNjM5MzgsLTAuMTA0MTIyMyAtMS43ODc3MDUsMC4wOTg2ODMgLTIuMTU1MjE4LDAuNjA2MDg1NCAtMC43OTc3NTIsMS4xMDE4MzIgMC40Mzc0NzcsMy4yNDE3NzkxIDEuMDMwOTY2LDQuMjcwNTcwMSAwLjEwODU2NSwwLjE4ODA0MiAwLjIwMjY1NSwwLjM1MTIxOSAwLjIzMjQxLDAuNDE5NTk4IDAuMTkzMDA1LDAuNDUzMDEgMC40NDMxMDYsMC43NTUyNzUgMC42MjU2NTYsMC45NzU5NTMgMC4wNTYyOSwwLjA2NzYgMC4xMTA5NzgsMC4xMzI4NzIgMC4xNTE5OTIsMC4xOTAzNzIgLTAuMzIxNjc1LDAuMDkwMTQgLTAuOTAwNjg4LDAuMjk3NjA0IC0wLjg0ODQxNiwxLjMzNDE2NSAtMC4wMTA0NiwwLjEyMTIxNyAtMC4wMzQ1OCwwLjM0NzMzNCAtMC4wNjc1NSwwLjYzMzI4MiAtMC4wMzY5OSwwLjE2MTYyMiAtMC4wNTYyOSwwLjM1NzQzNSAtMC4wODA0MiwwLjU5NTIwNyBtIDAuNzE1NzI1LC0xLjI1OTU3IGMgLTAuMDMyMTcsLTAuNjQ2NDkxIDAuMjE3MTMsLTAuNzE0MDkzIDAuNDgwMDk5LC0wLjc4NDgwMyBsIDAuMTA4NTY1LC0wLjAzMTg2IGEgMC44MDQxODU2OCwwLjc3NzAzMjQ5IDAgMCAwIDAuMTA3NzYxLDAuMDgwMDMgYyAwLjQ1ODM4NSwwLjI5MjE2NCAxLjI3MzAyNiwwLjMyNzEzMSAyLjQxODE4NiwwLjEwNDEyMyAtMC4xNjI0NDYsMC4xMzc1MzQgLTAuNDE3MzcyLDAuMzEwODEzIC0wLjc2NjM4OSwwLjQ2Njk5NiAtMC4zMjk3MTYsMC4xNDc2MzYgLTAuODgxMzg4LDAuMjU4NzUyIC0xLjQwNDkxMiwwLjI4Mjg0IC0wLjU3OTAxNCwwLjAyNjQyIC0wLjg3MzM0NiwtMC4wNjIxNiAtMC45NDMzMSwtMC4xMTczMzIgbSAwLjQ1ODM4NiwtNy4yMDM4NjggYyAtMC4wMDQ4LDAuMjcxOTYxNCAtMC4wNDM0MywwLjUxOTgzNDggLTAuMDg0NDQsMC43Nzc4MDk1IC0wLjA0NDIzLDAuMjc4MTc3NyAtMC4wOTAwNywwLjU2NTY3OTcgLTAuMTAyMTMxLDAuOTE0NTY3MyAtMC4wMTEyNiwwLjMzODc4NjIgMC4wMzIxNywwLjY5MTU1ODIgMC4wNzQ3OSwxLjAzMzQ1MzIgMC4wODYwNSwwLjY4OTIyOCAwLjE3MzcwNCwxLjM5ODY1OCAtMC4xNjY0NjcsMi4wOTg3NjQgYSAzLjIxNjc0MjcsMy4xMDgxMyAwIDAgMSAtMC4xNTExODcsLTAuMjk5MTU3IDYuNDMzNDg1NCw2LjIxNjI1OTkgMCAwIDAgLTAuMjYxMzYsLTAuNDc5NDI5IEMgMTYuNjgxOTI2LDExLjAzNTc1MiAxNS41MjMwOTQsOS4wMjYzNDYzIDE2LjExNTc3OSw4LjIwNzM1NDEgMTYuNDIxMzcsNy43ODU0MjU0IDE3LjE5NDk5Niw3Ljc2NzU1MzcgMTcuODY4MSw3Ljg0NzU4OCBtIDAuMTgzMzU0LDUuNDQ5MzI5IC0wLjA2ODM2LC0wLjA4MzE0IC0wLjAyODE1LC0wLjAzNDE5IGMgMC41ODM4MzgsLTAuOTMyNDM5IDAuNDY5NjQ0LC0xLjg1NDc3NyAwLjM2NzUxMywtMi42NzIyMTUgLTAuMDQxODIsLTAuMzM1Njc4IC0wLjA4MDQyLC0wLjY1MjcwNyAtMC4wNzA3NywtMC45NDk1MzM1IDAuMDEwNDUsLTAuMzE2MjUyMiAwLjA1MzA4LC0wLjU4NjY1OTUgMC4wOTQ4OSwtMC44NDg1MTk0IDAuMDUxNDcsLTAuMzIyNDY4NSAwLjEwNDU0NCwtMC42NTU4MTU1IDAuMDg5MjYsLTEuMDQ4OTkzOSBhIDAuNDgyNTExNDEsMC40NjYyMTk1IDAgMCAwIDAuMDA5NywtMC4xNDc2MzYyIEMgMTguNDA4NTExLDcuMTM1MDU0OSAxNy45NjI5OTMsNi4wMDY4MDM4IDE3LjA1NDI2Myw0Ljk4NTAwNiBhIDYuMjcyNjQ4Myw2LjA2MDg1MzQgMCAwIDAgLTIuMTYxNjUxLC0xLjU4NTE0NjMgNy40Nzg5MjY4LDcuMjI2NDAyMiAwIDAgMSAxLjYyNjg2NywtMC4xNDQ1MjggYyAxLjY1MDE4OSwwLjAzNTc0MyAyLjk1NTM4MywwLjYzMjUwNDQgMy44NzkzOTIsMS43NzM5NjUyIGEgMC44MDQxODU2OCwwLjc3NzAzMjQ5IDAgMCAxIDAuMDUzODgsMC4wNzc3MDMgQyAyMS4wMzQxNzgsNi4xNjA2NTYyIDIwLjIzMDc5Niw5Ljk4Mjg3OSAxOC4wNTA2NDksMTMuMjk2OTIzIE0gMTAuOTYwOTQ4LDguNTQ0NTkxOSBDIDEwLjk0MDg0Myw4LjY4NDQ1NzcgMTAuNzExNjUsOC44NzMyNzY2IDEwLjQ2MTU0OSw4Ljg3MzI3NjYgQSAwLjgwNDE4NTY4LDAuNzc3MDMyNDkgMCAwIDEgMTAuMzk2NDA5LDguODY4NjE0NCAwLjY0MzM0ODU0LDAuNjIxNjI1OTkgMCAwIDEgOS45ODk0OTE1LDguNjIzODQ5MiBDIDkuOTUyNDk5LDguNTc3MjI3MiA5Ljg5Mjk4OTIsOC40ODU1Mzc0IDkuOTA0MjQ3OCw4LjQwMjM5NDkgYSAwLjE3NjkyMDg1LDAuMTcwOTQ3MTUgMCAwIDEgMC4wNzQ3ODksLTAuMTE1Nzc3OCBjIDAuMDk0ODk0LC0wLjA2OTE1NiAwLjI4MzA3MzIsLTAuMDk0Nzk4IDAuNDkwNTUzMiwtMC4wNjY4MjUgMC4yNTQxMjMsMC4wMzQxODkgMC41MTYyODgsMC4xNDk5NjczIDAuNDkwNTU0LDAuMzI0Nzk5NiBtIDYuMzc3MTkyLC0wLjMxOTM2MDQgYyAwLjAwODgsMC4wNjIxNjMgLTAuMDM5NDEsMC4xNTU0MDY1IC0wLjEyMzA0LDAuMjQwODgwMSBhIDAuNTc5MDEzNjksMC41NTk0NjMzOSAwIDAgMSAtMC4zMjgxMDgsMC4xNzMyNzgyIDAuODA0MTg1NjgsMC43NzcwMzI0OSAwIDAgMSAtMC4wNjAzMSwwLjAwMzg5IGMgLTAuMjM1NjI2LDAgLTAuNDM1MDY0LC0wLjE4MTgyNTYgLTAuNDUwMzQ0LC0wLjI4ODI3OTEgLTAuMDE5MywtMC4xMzc1MzQ3IDAuMjEyMzA1LC0wLjI0MDg4IDAuNDUwMzQ0LC0wLjI3MzUxNTQgMC4yMzk2NDcsLTAuMDMyNjM1IDAuNDkyMTYyLDAuMDA2OTkgMC41MTE0NjIsMC4xNDM3NTEiCiAgICAgaWQ9InBhdGg5MTciCiAgICAgc3R5bGU9InN0cm9rZS13aWR0aDowLjc5MDQ5MyIgLz4KPC9zdmc+Cg==", "properties": { "host": { @@ -7744,7 +9309,7 @@ "collection_name": { "name": "collection_name", "title": "Collection Name", - "description": "The name of the collection that will be used for search.", + "description": "The name of the collection in Postgres Vector Store (PGVector).", "type": "string", "parameterType": "string", "value": null, @@ -7755,7 +9320,7 @@ "pre_delete_collection": { "name": "pre_delete_collection", "title": "Pre Delete Collection", - "description": "If set to true, then the collection will removed at the beginning.", + "description": "If enabled, deletes any existing collection with the same name before storing, discarding its previous contents. If disabled, new vectors are added to whatever the collection already contains.", "type": "string", "parameterType": "boolean", "value": "true", @@ -7779,7 +9344,7 @@ "embedding_path": { "name": "embedding_path", "title": "Embedding Path", - "description": "The name of the path to use for reading the embeddings.", + "description": "The name of the path to use for reading the embeddings. Its value must be a string representation of a list of floats, e.g. `[0.1, 0.2, ...]`.", "type": "string", "parameterType": "string", "value": "_embedding", @@ -7790,7 +9355,7 @@ "metadata_paths": { "name": "metadata_paths", "title": "Metadata Paths", - "description": "The comma separated list path names to be used as metadata. Empty name means all paths (except embedding source and embedding) will be used", + "description": "The comma-separated list of path names to store as metadata. An empty value means every path except the source and embedding paths is used.", "type": "string", "parameterType": "string", "value": "", @@ -7801,7 +9366,7 @@ "batch_processing_size": { "name": "batch_processing_size", "title": "Batch Processing Size", - "description": "The number of entries to be processed in batch.", + "description": "The number of entries to accumulate before writing them to the database as one batch.", "type": "string", "parameterType": "Long", "value": "100", @@ -8211,6 +9776,170 @@ "pluginType": "customtask", "relatedPlugins": [] }, + "cmem_plugin_s3-Upload": { + "pluginId": "cmem_plugin_s3-Upload", + "title": "Upload files to S3", + "categories": [ + "Uncategorized" + ], + "main_category": "Uncategorized", + "description": "Upload files to a bucket in S3 or S3-compatible object storage.", + "markdownDocumentation": "\nWrites the files that arrive on its input port into a bucket.\n\n## Ports\n\nFiles arrive on the input port, which is what a task in front of this one is\nfor - a download task, a task that produced a file, or anything else emitting\nfile entities. The task emits one entity per file it dealt with, carrying the\n`bucket` and the `key` it wrote, the `name` the key ends in, the `size`, the\n`etag` the service assigned and whether the file was `uploaded` or `skipped`.\n\nThat entity tag is a checksum of what now sits in the bucket, so a workflow can\nwrite it into a Knowledge Graph next to whatever else it records about the file.\n\n## Where the task sits\n\n**Download S3 objects** and every other download task in the fleet produce the\nfile entities this task accepts, which is how a workflow copies from one system\ninto object storage. **List S3 objects** reads a bucket back afterwards.\n\n## Caveats\n\n* **The key is the prefix plus the file's name**, and the name is all that a\n file entity carries - the key an object came from is not part of it. Two files\n with the same name therefore resolve to the same key. The second one is\n reported as a failure rather than written over the first.\n* **A large file is written in parts** and never held in memory as a whole. A\n file the project holds is first fetched to a temporary directory of the\n DataIntegration node, so the node needs room for it.\n* **Skip and Fail ask the service whether the key is taken** before writing, one\n request per file, and something writing that key in between is not noticed.\n Replace asks nothing and simply writes.\n* **The error handling decides whether one failure ends the run**, but the run\n ends in an error either way. Continuing only means the remaining files are\n written first.\n ", + "pluginIcon": "data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiIHN0YW5kYWxvbmU9Im5vIj8+CjxzdmcgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiB2aWV3Qm94PSIwIDAgMjQgMjQiIGZpbGw9Im5vbmUiIHN0cm9rZT0iY3VycmVudENvbG9yIiBzdHJva2Utd2lkdGg9IjEuNiIgc3Ryb2tlLWxpbmVjYXA9InJvdW5kIiBzdHJva2UtbGluZWpvaW49InJvdW5kIj4KICA8ZWxsaXBzZSBjeD0iMTIiIGN5PSI1IiByeD0iNyIgcnk9IjIuNiIvPgogIDxwYXRoIGQ9Ik01IDV2Ni40YzAgMS40NCAzLjEzIDIuNiA3IDIuNnM3LTEuMTYgNy0yLjZWNSIvPgogIDxwYXRoIGQ9Ik0xMiAyMS4ydi01LjYiLz4KICA8cGF0aCBkPSJNOS4yIDE4LjQgMTIgMTUuNmwyLjggMi44Ii8+Cjwvc3ZnPgo=", + "properties": { + "endpoint_url": { + "name": "endpoint_url", + "title": "Endpoint URL", + "description": "The address of the service, e.g. `https://minio.example.org`. Leave empty for Amazon S3, which is then addressed by region.", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "region": { + "name": "region", + "title": "Region", + "description": "The region the bucket lives in. Amazon S3 needs the bucket's own region; most other implementations accept any value and ignore it.", + "type": "string", + "parameterType": "string", + "value": "us-east-1", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "credentials": { + "name": "credentials", + "title": "Credentials", + "description": "Where the credentials come from.", + "type": "string", + "parameterType": "string", + "value": "explicit", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "access_key_id": { + "name": "access_key_id", + "title": "Access key ID", + "description": "The access key to sign requests with.", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "secret_access_key": { + "name": "secret_access_key", + "title": "Secret access key", + "description": "The secret belonging to the access key.", + "type": "string", + "parameterType": "password", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "bucket": { + "name": "bucket", + "title": "Bucket", + "description": "The bucket the files are written to. It has to exist already.", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "prefix": { + "name": "prefix", + "title": "Key prefix", + "description": "Written in front of each file's name to make the key, e.g. `incoming/`. Empty writes the names to the top of the bucket.", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "on_existing": { + "name": "on_existing", + "title": "When the key is already taken", + "description": "What happens to an object the bucket already holds under the key this task would write. Replacing is what makes a repeated run leave the bucket as it was after the first one.", + "type": "string", + "parameterType": "string", + "value": "replace", + "advanced": false, + "visibleInDialog": true, + "properties": {} + } + }, + "properties_advanced": { + "session_token": { + "name": "session_token", + "title": "Session token", + "description": "The session token of a temporary credential. Needed only for keys issued by a token service, which expire.", + "type": "string", + "parameterType": "password", + "value": "", + "advanced": true, + "visibleInDialog": true, + "properties": {} + }, + "addressing_style": { + "name": "addressing_style", + "title": "Bucket addressing style", + "description": "How the bucket is named in a request.", + "type": "string", + "parameterType": "string", + "value": "auto", + "advanced": true, + "visibleInDialog": true, + "properties": {} + }, + "ssl_verify": { + "name": "ssl_verify", + "title": "Verify TLS connection", + "description": "If enabled, the certificate of the service is verified. Disable only when connecting to a service with a self-signed or otherwise untrusted certificate.", + "type": "string", + "parameterType": "boolean", + "value": "true", + "advanced": true, + "visibleInDialog": true, + "properties": {} + }, + "on_error": { + "name": "on_error", + "title": "On error", + "description": "What happens when a single object cannot be transferred. Either way the workflow ends in an error.", + "type": "string", + "parameterType": "string", + "value": "stop", + "advanced": true, + "visibleInDialog": true, + "properties": {} + } + }, + "actions": { + "test_connection": { + "label": "Test connection", + "description": "Reach the service, say which identity it signs as and whether the bucket can be listed.", + "icon": null + } + }, + "required": [], + "distanceMeasureRange": null, + "backendType": "python", + "is_deprecated": false, + "tags": [ + "WorkflowTask", + "PythonPlugin" + ], + "pluginType": "customtask", + "relatedPlugins": [] + }, "cmem_plugin_git-Upload": { "pluginId": "cmem_plugin_git-Upload", "title": "Upload Git files", @@ -8429,14 +10158,14 @@ "Uncategorized" ], "main_category": "Uncategorized", - "description": "Upload files to a given SSH instance.", - "markdownDocumentation": "\nThis workflow task uploads files to a given SSH instance.\n\nBy providing the hostname, username, port and authentication method, you can specify the\nfolder the data should be uploaded to.\n\n#### Authentication Methods:\n* **Password:** Only the password will be used for authentication. The private key field is\nignored, even if filled.\n* **Key:** The private key will be used for authentication. If the key is encrypted, the password\nwill be used to decrypt it.\n\n#### Note:\n* If a connection cannot be established within 20 seconds, a timeout occurs.\n* Currently supported key types are: RSA, ECDSA, Ed25519.\n ", + "description": "Upload files to an SSH server.", + "markdownDocumentation": "\nUploads the files it receives to a directory on an SSH server.\n\nFiles arrive on the input port as file entities and are written into the configured\ndirectory under their plain file name. The task has no output port: it ends the branch\nof the workflow it sits in and hands nothing on to a following task.\n\n**Download SSH files** covers the other direction, and **Execute commands via SSH**\nruns a command on the uploaded files once they are in place.\n\n#### Caveats\n\n* Content that is gzip compressed is unpacked before it is sent. Such a file arrives on\nthe server uncompressed, under an unchanged name that still ends in `.gz`.\n* A file of the same name already on the server is overwritten without warning, and so\nare files of the same name coming from different folders.\n* The first failing upload aborts the task. Files uploaded before that stay on the\nserver, and nothing is rolled back.\n* The target directory has to exist. It is not created.\n* The task logs in with the configured credentials only, and offers no key of the\nmachine it runs on. The host key of the server in turn is accepted as presented and\nnever checked against a known hosts list.\n* Establishing the connection fails after 20 seconds.\n ", "pluginIcon": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTQuNSA5Yy0uNiAwLTEgLjUtMSAxdjEuNzVjMCAuNS40IDEgMSAxSDd2Ljc1SDMuNVYxNWg0Yy42IDAgMS0uNSAxLTF2LTEuNzVjMC0uNS0uNC0xLTEtMUg1di0uNzVoMy41Vjl6bTYgMGMtLjYgMC0xIC41LTEgMXYxLjc1YzAgLjUuNCAxIDEgMUgxM3YuNzVIOS41VjE1aDRjLjYgMCAxLS41IDEtMXYtMS43NWMwLS41LS40LTEtMS0xSDExdi0uNzVoMy41Vjl6bTUgMHY2SDE3di0yLjVoMlYxNWgxLjVWOUgxOXYyaC0yVjl6Ii8+PC9zdmc+", "properties": { "hostname": { "name": "hostname", "title": "Hostname", - "description": "Hostname to connect to. Usually in the form of an IP address", + "description": "Host name or IP address of the SSH server.", "type": "string", "parameterType": "string", "value": null, @@ -8447,7 +10176,7 @@ "port": { "name": "port", "title": "Port", - "description": "The port on which the connection will be tried on. Default is 22.", + "description": "TCP port the SSH server listens on.", "type": "string", "parameterType": "Long", "value": "22", @@ -8458,7 +10187,7 @@ "username": { "name": "username", "title": "Username", - "description": "The username with which a connection will be instantiated.", + "description": "Account to log in as.", "type": "string", "parameterType": "string", "value": null, @@ -8469,7 +10198,7 @@ "authentication_method": { "name": "authentication_method", "title": "Authentication method", - "description": "The method that is used to connect to the SSH server.", + "description": "How the task authenticates against the server.", "type": "string", "parameterType": "string", "value": "password", @@ -8480,7 +10209,7 @@ "private_key": { "name": "private_key", "title": "Private key", - "description": "Your private key to connect via SSH.", + "description": "Private key in PEM format, used when the authentication method is Key. RSA, ECDSA and Ed25519 keys are supported.", "type": "string", "parameterType": "password", "value": "", @@ -8491,7 +10220,7 @@ "password": { "name": "password", "title": "Password", - "description": "Depending on your authentication method this will either be used toconnect via password to SSH, or to decrypt the SSH private key", + "description": "Password of the account, or the passphrase of the private key when the authentication method is Key.", "type": "string", "parameterType": "password", "value": "", @@ -8502,7 +10231,7 @@ "path": { "name": "path", "title": "Path", - "description": "The currently selected path within your SSH instance. Auto-completion starts from user home folder, use '..' for parent directory or '/' for root directory.", + "description": "Remote directory the files are written to. Autocompletion starts in the home directory of the account, use '..' for the parent directory or '/' for the root directory.", "type": "string", "parameterType": "string", "value": "", @@ -8535,13 +10264,13 @@ ], "main_category": "Uncategorized", "description": "Use a JSON schema to validate entities or a JSON dataset.", - "markdownDocumentation": "[JSON Schema](https://json-schema.org/) specifies a JSON-based format to\ndefine the structure of JSON data for validation, documentation, and interaction control.\nIt provides a contract for the JSON data required by a given application.\n\nThis workflow task can validate incoming entities or a stand-alone JSON dataset by using a\nJSON Schema specification.\n\nThe used JSON Schema needs to be provided as a JSON Dataset in the project.\n\n### Input Modes\n\nThe plugin supports two input modes for validation:\n\n1. **Validate Entities**: Validates entities received from the input port in the workflow.\n2. **Validate JSON Dataset**: Validates a JSON dataset stored in the project.\n - If the JSON dataset is a JSON array, the schema will validate each object inside the array.\n - If the JSON dataset is a JSON object, it will be validated against the schema directly.\n\nValidated data objects can be sent to an output port for further processing in the workflow\nor saved in a JSON dataset in the project.\n\n### Output Modes\n\n1. **Valid JSON objects sent to Output Port**: Valid JSON objects can be sent as entities\n to the output port.\n2. **Saved in JSON Dataset**: Valid JSON objects can be stored in a specified JSON dataset\n in the project.\n\n### Error Handling\n\nThe task can either:\n\n- Fail instantly if there is a data violation, halting the workflow.\n- Provide warnings in the workflow report, allowing follow-up tasks to run based on the\n validated data.\n\nThe error handling behavior is configurable through the `Fail on violations` parameter.\n", + "markdownDocumentation": "[JSON Schema](https://json-schema.org/) specifies a JSON-based format to\ndefine the structure of JSON data for validation, documentation, and interaction control.\nIt provides a contract for the JSON data required by a given application.\n\nThis workflow task can validate incoming entities or a stand-alone JSON dataset by using a\nJSON Schema specification.\n\nThe used JSON Schema needs to be provided as a JSON Dataset in the project.\n\n### Input Modes\n\nEntities arrive on the input port by default. Switching the input mode to a JSON dataset\nremoves the input port, and the resources to validate are instead read from a JSON dataset in\nthe project. A JSON array in that dataset has each of its objects validated individually; a\nJSON object is validated directly.\n\n### Output Modes\n\nValid JSON objects are sent to an output port by default, ready for further processing in the\nworkflow. Switching the output mode to a JSON dataset removes the output port, and the valid\nJSON objects are instead saved to a JSON dataset in the project, replacing its existing content.\n\n### Error Handling\n\nEvery entity or object is validated regardless of the outcome. The task can then either:\n\n- Fail once validation completes if any entity has violations, halting the workflow.\n- Report violations only as warnings, allowing follow-up tasks to run based on the\n validated data.\n\nThe error handling behavior is configurable through the **Fail on violations** parameter.\n", "pluginIcon": "data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiIHN0YW5kYWxvbmU9Im5vIj8+CjxzdmcKICAgdmlld0JveD0iMCAwIDI0IDI0IgogICB2ZXJzaW9uPSIxLjEiCiAgIGlkPSJzdmc4MjgiCiAgIHNvZGlwb2RpOmRvY25hbWU9ImNtZW0tcGx1Z2luLXZhbGlkYXRpb24uc3ZnIgogICBpbmtzY2FwZTp2ZXJzaW9uPSIxLjEuMiAoYjhlMjViZTgsIDIwMjItMDItMDUpIgogICB4bWxuczppbmtzY2FwZT0iaHR0cDovL3d3dy5pbmtzY2FwZS5vcmcvbmFtZXNwYWNlcy9pbmtzY2FwZSIKICAgeG1sbnM6c29kaXBvZGk9Imh0dHA6Ly9zb2RpcG9kaS5zb3VyY2Vmb3JnZS5uZXQvRFREL3NvZGlwb2RpLTAuZHRkIgogICB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciCiAgIHhtbG5zOnN2Zz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPgogIDxkZWZzCiAgICAgaWQ9ImRlZnM4MzIiIC8+CiAgPHNvZGlwb2RpOm5hbWVkdmlldwogICAgIGlkPSJuYW1lZHZpZXc4MzAiCiAgICAgcGFnZWNvbG9yPSIjZmZmZmZmIgogICAgIGJvcmRlcmNvbG9yPSIjNjY2NjY2IgogICAgIGJvcmRlcm9wYWNpdHk9IjEuMCIKICAgICBpbmtzY2FwZTpwYWdlc2hhZG93PSIyIgogICAgIGlua3NjYXBlOnBhZ2VvcGFjaXR5PSIwLjAiCiAgICAgaW5rc2NhcGU6cGFnZWNoZWNrZXJib2FyZD0iMCIKICAgICBzaG93Z3JpZD0iZmFsc2UiCiAgICAgaW5rc2NhcGU6em9vbT0iMzUuMjkxNjY3IgogICAgIGlua3NjYXBlOmN4PSIxMiIKICAgICBpbmtzY2FwZTpjeT0iMTIiCiAgICAgaW5rc2NhcGU6d2luZG93LXdpZHRoPSIxOTIwIgogICAgIGlua3NjYXBlOndpbmRvdy1oZWlnaHQ9IjEwMjciCiAgICAgaW5rc2NhcGU6d2luZG93LXg9IjE3MjgiCiAgICAgaW5rc2NhcGU6d2luZG93LXk9IjI1IgogICAgIGlua3NjYXBlOndpbmRvdy1tYXhpbWl6ZWQ9IjAiCiAgICAgaW5rc2NhcGU6Y3VycmVudC1sYXllcj0ic3ZnODI4IiAvPgogIDxnCiAgICAgaWQ9Imc5MTYiCiAgICAgdHJhbnNmb3JtPSJtYXRyaXgoMC44ODQ4MjMxNywwLDAsMC44ODQ4MjMxNywxLjMxNTQ3MjMsMS41MDg2MzIyKSI+CiAgICA8cGF0aAogICAgICAgZD0iTSAxNi41Myw5Ljc4IEEgMC43NSwwLjc1IDAgMCAwIDE1LjQ3LDguNzIgTCAxMSwxMy4xOSA5LjAzLDExLjIyIGEgMC43NSwwLjc1IDAgMCAwIC0xLjA2LDEuMDYgbCAyLjUsMi41IGEgMC43NSwwLjc1IDAgMCAwIDEuMDYsMCB6IgogICAgICAgaWQ9InBhdGg4MjQiIC8+CiAgICA8cGF0aAogICAgICAgZD0ibSAxMi41NCwwLjYzNyA4LjI1LDIuNjc1IEEgMS43NSwxLjc1IDAgMCAxIDIyLDQuOTc2IFYgMTAgYyAwLDYuMTkgLTMuNzcxLDEwLjcwNCAtOS40MDEsMTIuODMgYSAxLjcsMS43IDAgMCAxIC0xLjE5OCwwIEMgNS43NywyMC43MDUgMiwxNi4xOSAyLDEwIFYgNC45NzYgQyAyLDQuMjE4IDIuNDg5LDMuNTQ2IDMuMjEsMy4zMTIgbCA4LjI1LC0yLjY3NSBhIDEuNzUsMS43NSAwIDAgMSAxLjA4LDAgTSAxMS45MjMsMi4wNjMgMy42NzMsNC43MzkgQSAwLjI1LDAuMjUgMCAwIDAgMy41LDQuOTc2IFYgMTAgYyAwLDUuNDYgMy4yOCw5LjQ4MyA4LjQzLDExLjQyNiBhIDAuMiwwLjIgMCAwIDAgMC4xNCwwIEMgMTcuMjIsMTkuNDgzIDIwLjUsMTUuNDYxIDIwLjUsMTAgViA0Ljk3NiBBIDAuMjUsMC4yNSAwIDAgMCAyMC4zMjcsNC43MzkgbCAtOC4yNSwtMi42NzYgYSAwLjI1LDAuMjUgMCAwIDAgLTAuMTU0LDAiCiAgICAgICBpZD0icGF0aDgyNiIgLz4KICA8L2c+Cjwvc3ZnPgo=", "properties": { "source_mode": { "name": "source_mode", "title": "Source / Input Mode", - "description": "", + "description": "Selects where entities to validate come from: the input port or a JSON dataset.", "type": "string", "parameterType": "string", "value": "entities", @@ -8552,7 +10281,7 @@ "target_mode": { "name": "target_mode", "title": "Target / Output Mode", - "description": "", + "description": "Selects where valid JSON objects are written to: the output port or a JSON dataset.", "type": "string", "parameterType": "string", "value": "entities", @@ -8574,7 +10303,7 @@ "fail_on_violations": { "name": "fail_on_violations", "title": "Fail on violations", - "description": "If enabled, the task will fail on the first data violation.", + "description": "If enabled, the workflow fails once validation completes if any entity has violations. All entities are validated either way; disabling this instead reports the violations as warnings.", "type": "string", "parameterType": "boolean", "value": "false", @@ -8587,7 +10316,7 @@ "source_dataset": { "name": "source_dataset", "title": "Source JSON Dataset", - "description": "This dataset holds the resources you want to validate.", + "description": "This dataset holds the resources you want to validate. Required when Source / Input Mode is set to dataset; leave it empty when using entities.", "type": "string", "parameterType": "string", "value": "", @@ -8598,7 +10327,7 @@ "target_dataset": { "name": "target_dataset", "title": "Target JSON Dataset", - "description": "This dataset will be used to store the valid JSON objects after validation.", + "description": "This dataset stores the valid JSON objects after validation, replacing any existing content. Required when Target / Output Mode is set to dataset; leave it empty when using entities.", "type": "string", "parameterType": "string", "value": "", @@ -8629,7 +10358,7 @@ ], "main_category": "Uncategorized", "description": "Use SHACL shapes to validate resources in a Knowledge Graph.", - "markdownDocumentation": "\nStart a graph validation process which verifies, that resources in a specific graph are valid\naccording to the node shapes in a shape catalog graph.\n", + "markdownDocumentation": "\nStarts a graph validation process which verifies that resources in a specific graph are valid\naccording to the node shapes in a shape catalog graph. The task waits for the validation to\nfinish before completing.\n\nThe task has no input port. Violations found during validation are always summarized in the\nworkflow report, and can also be materialized into a result graph in the project. Sending each\nviolation as an entity to an output port is optional; when that is disabled, the task has no\noutput port either.\n\n### Error Handling\n\nEvery resource is validated regardless of the outcome. The task can then either:\n\n- Fail once validation completes if any resource has violations, halting the workflow.\n- Report violations only as warnings, allowing follow-up tasks to run based on the results.\n\nThe error handling behavior is configurable through the **Fail workflow on violations**\nparameter.\n", "pluginIcon": "data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiIHN0YW5kYWxvbmU9Im5vIj8+CjxzdmcKICAgdmlld0JveD0iMCAwIDI0IDI0IgogICB2ZXJzaW9uPSIxLjEiCiAgIGlkPSJzdmc4MjgiCiAgIHNvZGlwb2RpOmRvY25hbWU9ImNtZW0tcGx1Z2luLXZhbGlkYXRpb24uc3ZnIgogICBpbmtzY2FwZTp2ZXJzaW9uPSIxLjEuMiAoYjhlMjViZTgsIDIwMjItMDItMDUpIgogICB4bWxuczppbmtzY2FwZT0iaHR0cDovL3d3dy5pbmtzY2FwZS5vcmcvbmFtZXNwYWNlcy9pbmtzY2FwZSIKICAgeG1sbnM6c29kaXBvZGk9Imh0dHA6Ly9zb2RpcG9kaS5zb3VyY2Vmb3JnZS5uZXQvRFREL3NvZGlwb2RpLTAuZHRkIgogICB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciCiAgIHhtbG5zOnN2Zz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPgogIDxkZWZzCiAgICAgaWQ9ImRlZnM4MzIiIC8+CiAgPHNvZGlwb2RpOm5hbWVkdmlldwogICAgIGlkPSJuYW1lZHZpZXc4MzAiCiAgICAgcGFnZWNvbG9yPSIjZmZmZmZmIgogICAgIGJvcmRlcmNvbG9yPSIjNjY2NjY2IgogICAgIGJvcmRlcm9wYWNpdHk9IjEuMCIKICAgICBpbmtzY2FwZTpwYWdlc2hhZG93PSIyIgogICAgIGlua3NjYXBlOnBhZ2VvcGFjaXR5PSIwLjAiCiAgICAgaW5rc2NhcGU6cGFnZWNoZWNrZXJib2FyZD0iMCIKICAgICBzaG93Z3JpZD0iZmFsc2UiCiAgICAgaW5rc2NhcGU6em9vbT0iMzUuMjkxNjY3IgogICAgIGlua3NjYXBlOmN4PSIxMiIKICAgICBpbmtzY2FwZTpjeT0iMTIiCiAgICAgaW5rc2NhcGU6d2luZG93LXdpZHRoPSIxOTIwIgogICAgIGlua3NjYXBlOndpbmRvdy1oZWlnaHQ9IjEwMjciCiAgICAgaW5rc2NhcGU6d2luZG93LXg9IjE3MjgiCiAgICAgaW5rc2NhcGU6d2luZG93LXk9IjI1IgogICAgIGlua3NjYXBlOndpbmRvdy1tYXhpbWl6ZWQ9IjAiCiAgICAgaW5rc2NhcGU6Y3VycmVudC1sYXllcj0ic3ZnODI4IiAvPgogIDxnCiAgICAgaWQ9Imc5MTYiCiAgICAgdHJhbnNmb3JtPSJtYXRyaXgoMC44ODQ4MjMxNywwLDAsMC44ODQ4MjMxNywxLjMxNTQ3MjMsMS41MDg2MzIyKSI+CiAgICA8cGF0aAogICAgICAgZD0iTSAxNi41Myw5Ljc4IEEgMC43NSwwLjc1IDAgMCAwIDE1LjQ3LDguNzIgTCAxMSwxMy4xOSA5LjAzLDExLjIyIGEgMC43NSwwLjc1IDAgMCAwIC0xLjA2LDEuMDYgbCAyLjUsMi41IGEgMC43NSwwLjc1IDAgMCAwIDEuMDYsMCB6IgogICAgICAgaWQ9InBhdGg4MjQiIC8+CiAgICA8cGF0aAogICAgICAgZD0ibSAxMi41NCwwLjYzNyA4LjI1LDIuNjc1IEEgMS43NSwxLjc1IDAgMCAxIDIyLDQuOTc2IFYgMTAgYyAwLDYuMTkgLTMuNzcxLDEwLjcwNCAtOS40MDEsMTIuODMgYSAxLjcsMS43IDAgMCAxIC0xLjE5OCwwIEMgNS43NywyMC43MDUgMiwxNi4xOSAyLDEwIFYgNC45NzYgQyAyLDQuMjE4IDIuNDg5LDMuNTQ2IDMuMjEsMy4zMTIgbCA4LjI1LC0yLjY3NSBhIDEuNzUsMS43NSAwIDAgMSAxLjA4LDAgTSAxMS45MjMsMi4wNjMgMy42NzMsNC43MzkgQSAwLjI1LDAuMjUgMCAwIDAgMy41LDQuOTc2IFYgMTAgYyAwLDUuNDYgMy4yOCw5LjQ4MyA4LjQzLDExLjQyNiBhIDAuMiwwLjIgMCAwIDAgMC4xNCwwIEMgMTcuMjIsMTkuNDgzIDIwLjUsMTUuNDYxIDIwLjUsMTAgViA0Ljk3NiBBIDAuMjUsMC4yNSAwIDAgMCAyMC4zMjcsNC43MzkgbCAtOC4yNSwtMi42NzYgYSAwLjI1LDAuMjUgMCAwIDAgLTAuMTU0LDAiCiAgICAgICBpZD0icGF0aDgyNiIgLz4KICA8L2c+Cjwvc3ZnPgo=", "properties": { "context_graph": { @@ -8668,7 +10397,7 @@ "clear_result_graph": { "name": "clear_result_graph", "title": "Clear result graph before validation", - "description": "", + "description": "If enabled, the existing content of the result graph is deleted before validation starts. Has no effect when Result graph is left empty.", "type": "string", "parameterType": "boolean", "value": "false", @@ -8679,7 +10408,7 @@ "fail_on_violations": { "name": "fail_on_violations", "title": "Fail workflow on violations", - "description": "", + "description": "If enabled, the workflow fails once validation completes if any resource has violations. All resources are validated either way; disabling this instead reports the violations as warnings.", "type": "string", "parameterType": "boolean", "value": "false", @@ -8690,7 +10419,7 @@ "output_results": { "name": "output_results", "title": "Output violations as entities", - "description": "", + "description": "If enabled, each violation is sent as an entity to an output port for further processing in the workflow. Disabling this removes the output port.", "type": "string", "parameterType": "boolean", "value": "true", @@ -8703,7 +10432,7 @@ "sparql_query": { "name": "sparql_query", "title": "Resource Selection Query", - "description": "The query to select the resources to validate. Use {{context_graph}} as a placeholder for the select context graph for validation.", + "description": "The query to select the resources to validate. Use `{{context_graph}}` as a placeholder for the selected Context Graph.", "type": "string", "parameterType": "code-sparql", "value": "SELECT DISTINCT ?resource\nFROM <{{context_graph}}>\nWHERE { ?resource a ?class . FILTER isIRI(?resource) }\n", @@ -8734,7 +10463,7 @@ ], "main_category": "Uncategorized", "description": "Validates the consistency of an OWL ontology.", - "markdownDocumentation": "A task validating the consistency of an OWL ontology and generating an explanation if inconsistencies are found. \nThe plugin outputs the explanation as text in Markdown format on the path \"markdown\", the ontology IRI on the path \n\"ontology_graph_iri\", and, if enabled, the valid OWL2 profiles on the path \"valid_profiles\" as a comma-separated string.\n\n## Options\n\n### Ontology graph IRI\n\nThe IRI of the input ontology graph. The graph IRI is selected from a list of graphs of type`owl:Ontology`.\n\n### Ignore missing imports\n\nIf enabled, missing imports (`owl:imports`) in the input graphs are ignored.\n\n### Reasoner\n\nThe following reasoner options are supported: \n- [ELK](https://code.google.com/p/elk-reasoner/) (elk)\n- [Expression Materializing Reasoner](http://static.javadoc.io/org.geneontology/expression-materializing-reasoner/0.1.3/org/geneontology/reasoner/ExpressionMaterializingReasoner.html) (emr)\n- [HermiT](http://www.hermit-reasoner.com/) (hermit)\n- [JFact](http://jfact.sourceforge.net/) (jfact)\n- [Structural Reasoner](http://owlcs.github.io/owlapi/apidocs_4/org/semanticweb/owlapi/reasoner/structural/StructuralReasoner.html) (structural)\n- [Whelk](https://github.com/balhoff/whelk) (whelk)\n\n\u26a0\ufe0f Only **HermiT** and **JFact** are recommended for consistency validation, since they are the\nonly reasoner options here that are complete OWL DL reasoners capable of generating explanations.\nThe remaining options are kept for backwards compatibility.\n\n### Produce output graph\n\nIf enabled, an explanation graph is created.\n\n### Output graph IRI\n\nThe IRI of the output graph for the reasoning result.\n\n\u26a0\ufe0f Existing graphs will be overwritten.\n\n### Write markdown explanation file\n\nIf enabled, an explanation markdown file is written to the project.\n\n### Output filename\n\nThe filename of the Markdown file with the explanation of inconsistencies.\n\n\u26a0\ufe0f Existing files will be overwritten.\n\n### Stop at inconsistencies\nRaise an error if inconsistencies are found. If enabled, the plugin does not output entities.\n\n### Validate OWL2 profiles\n\nValidate the input ontology against OWL profiles (DL, EL, QL, RL, and Full) and annotate the result graph.\n\n### Mode\nMode _inconsistency_ generates an explanation for an inconsistent ontology. \nMode _unsatisfiability_ generates explanations for many unsatisfiable classes at once.\n\n### Output entities\n\nOutput entities. The plugin outputs the explanation as text in Markdown format on the path \"markdown\", the ontology IRI\non the path \"ontology_graph_iri\", the reasoner option on the path \"reasoner\", and, if enabled, the valid OWL2 profiles\non the path \"valid_profiles\".\n\n### Maximum RAM Percentage\n\nMaximum heap size for the Java virtual machine in the DI container running the reasoning process.\n\n\u26a0\ufe0f Setting the percentage too high may result in an out of memory error.\n", + "markdownDocumentation": "A task validating the consistency of an OWL ontology and generating an explanation if inconsistencies are found. \nThe plugin outputs the explanation as text in Markdown format on the path \"markdown\", the ontology IRI on the path \n\"ontology_graph_iri\", and, if enabled, the valid OWL2 profiles on the path \"valid_profiles\" as a comma-separated string.\n\n## Options\n\n### Ontology graph IRI\n\nThe IRI of the input ontology graph. The graph IRI is selected from a list of graphs of type`owl:Ontology`.\n\n### Ignore missing imports\n\nIf enabled, missing imports (`owl:imports`) in the input graphs are ignored.\n\n### Reasoner\n\nThe following reasoner options are supported: \n- [ELK](https://code.google.com/p/elk-reasoner/) (elk)\n- [Expression Materializing Reasoner](http://static.javadoc.io/org.geneontology/expression-materializing-reasoner/0.1.3/org/geneontology/reasoner/ExpressionMaterializingReasoner.html) (emr)\n- [HermiT](http://www.hermit-reasoner.com/) (hermit)\n- [JFact](http://jfact.sourceforge.net/) (jfact)\n- [Structural Reasoner](http://owlcs.github.io/owlapi/apidocs_4/org/semanticweb/owlapi/reasoner/structural/StructuralReasoner.html) (structural)\n- [Whelk](https://github.com/balhoff/whelk) (whelk)\n\n\u26a0\ufe0f Only **HermiT** and **JFact** are recommended for consistency validation, since they are the\nonly reasoner options here that are complete OWL DL reasoners capable of generating explanations.\nThe remaining options are kept for backwards compatibility.\n\n### Produce output graph\n\nIf enabled, an explanation graph is created.\n\n### Output graph IRI\n\nThe IRI of the output graph for the reasoning result.\n\n\u26a0\ufe0f Existing graphs will be overwritten.\n\nThe output graph holds the explanation axioms plus the validation result and is declared a\n`void:Dataset`. It is not declared an `owl:Ontology`: it does not contain an ontology of its\nown, it describes the validated one.\n\n### Write markdown explanation file\n\nIf enabled, an explanation markdown file is written to the project.\n\n### Output filename\n\nThe filename of the Markdown file with the explanation of inconsistencies.\n\n\u26a0\ufe0f Existing files will be overwritten.\n\n### Stop at inconsistencies\nRaise an error if inconsistencies are found. If enabled, the plugin does not output entities.\n\n### Validate OWL2 profiles\n\nValidate the input ontology against OWL profiles (DL, EL, QL, RL, and Full) and annotate the result graph.\n\n### Mode\nMode _inconsistency_ generates an explanation for an inconsistent ontology. \nMode _unsatisfiability_ generates explanations for many unsatisfiable classes at once.\n\n### Output entities\n\nOutput entities. The plugin outputs the explanation as text in Markdown format on the path \"markdown\", the ontology IRI\non the path \"ontology_graph_iri\", the reasoner option on the path \"reasoner\", and, if enabled, the valid OWL2 profiles\non the path \"valid_profiles\".\n\n### Maximum RAM Percentage\n\nMaximum heap size for the Java virtual machine in the DI container running the reasoning process.\n\n\u26a0\ufe0f Setting the percentage too high may result in an out of memory error.\n", "pluginIcon": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIxOTIiIGhlaWdodD0iMTkyIiB2aWV3Qm94PSIwIC03MCAzMTggNjYwIj4KICAgIDxwYXRoIGZpbGw9ImN1cnJlbnRDb2xvciIgZD0iTTE3Ny4xNjYgOTguNTI0Yy0uNDc4LTEyLjI2My0xNS4xNjMtMTguMzgtMjIuMjA5LTIzLjEyOGMtNS42NzMtMy44MjMtMzguNiAxNC43NzUtNTEuNTI0IDIzLjE3NkM4Ny41MDcgMTA4LjkyNCA5MS4xNyAxMzEuMzggOTEuMTcgMTMxLjM4Yy00LjMgNC42MTgtNi40NDcgNy4xMi03LjgwMyAxNC44MWMtLjk1NiA1LjQxNSAxLjM3NCAxMC45MyAyLjU0OCAxNi4wODVjLjk0OCA0LjE2Ny43OTYgNy4zMjYuNzk2IDcuMzI2Yy0xMC41MSAzLjE4NS0xOC4xNTUgMTQuODEtMTguMTU1IDE0LjgxSDY1LjM3Yy0yLjA3LTMuMDI2LTMuODIyLTI0LjUyNS0uMzE5LTMwLjQxN2MyNS44MTUtMTkuMzMuNDc4LTM5LjMzNi0yNS43OTktNTAuMTY1Yy02LjE1LTIuNTM1LTEwLjcxNS0uODQzLTE1LjQ0OC4xNTljLTIxLjgxNyA0LjYxOC0zLjk4IDU2LjA1Ny0zLjk4IDU2LjA1N3Y2LjA1MnMtMjEuMzM5LTQ1LjM3LTE5Ljc0LTU3LjI0OWMxLjE2OC04LjY3MSA5LjQ5Ni0xNi44MjUgMTcuODItMTguNzMzYzIuODgzLS42Ni4wMjktOC4wNjcgMS41OTktMTUuNzYzYzIuMDIzLTkuOTE2IDExLjAwOC0yMi41MSAxMS4wMDgtMjIuNTFjNS4yNSA2LjI5IDEyLjI3MSAzMy4xMyAxMi4yNzEgMzMuMTNsMTguNDU2LjA2OHM3OC44MjYtMzQuMzYyIDg4LjYyNy0zNi45MjhMMTY5LjAyMyAwYzEyLjk5NCA4Ljc5NiAyMC44MTQgMjYuMjQzIDIxLjQxNyA0OC41NjVjMCAwIDE4LjUwNCAyLjM3OSAyOC45NTggMTQuMTc3YzIxLjExNiAyMy44MyAyNC42OTIgNTguMDEgNTQuMjk3IDgxLjQ0NWMxMi43MDkgMTAuMDYgMjQuMTkgMTguMTIzIDMxLjg0MiAzNC43NzdjMTcuMzMxIDM3LjcyNCAxNi45MDggOTkuNS04LjIwNiAxMzkuMDEyYzAgMC02Mi43NDggOTIuNjc0LTU4Ljg4NyAxMTYuODA4YzMuODYyIDI0LjEzNCAyOS45MjYgMjUuNTgyIDY2LjEyNyAyNS41ODJsLTI3LjExOSAzNy42MjNjLTUyLjI3OSAxMC4zNzYtMTE0LjY0OCAzNS4wMjEtMTA4LjUxNC0yNC4xMDhjMCAwLTk5LjUyOCA1NS4xNy0xMzcuMDggMjkuOTI2Yy0xMy40MDUtOS4wMS0zOS41OC04OC43OTgtMTkuNzktMTQ3LjQ4OWM0LjgzNS0xNC4zMzcgMTUuMTAyLTMzLjQ1MyAyOC45Ni00OS40NDRjMzEuMzc1LTM2LjIgNTUuMDI2LTUyLjYxMiA3Mi4zODMtNjguNjAzYzEzLjMyNy0xMi4yOCAyNC43NTgtMTYuMzI2IDI5LjQ2My0yNC4wNzFjOS44Mi0xNi4xNjcgMS45MDgtMjguODcxIDEuOTA4LTI4Ljg3MWMxOC44NTYtNy42MyAzMS41NS0xOS41MjUgMzYuMTQ3LTI3Ljg1NmM1LjI0Mi05LjUwNC01LjYzOC04Ljc4OC01LjYzOC04Ljc4OHMxMC40MjItMTIuNDQ3IDEzLjEtMjIuNzE2Yy40NDUtMS43MTEtMy4xNjMtNC43NzMtMi44Mi02LjQ2N2MxLjIxMi01Ljk4IDIuMTU2LTE0LjI2LjY2NC0xNi43NDdjLTQuNzktNy45ODItOS4wNy00LjIzMS05LjA3LTQuMjMxbS03OC4wOTggMzMzLjdjNjAuMjQyLTYuMzQ4IDczLjU3MS0zMi4xOTMgNzMuNDEtMzMuMzE5YzAgMC0yNi43NTMtLjc1LTMwLjM0Ny0xLjQxMmMtNS4wMDktLjkyMy02LjU4Ni0zLjMzLTYuNTg2LTMuMzNjNjcuOTI2IDEuNzc2IDk2LjQyLTM5LjQ4MiAxMTYuNDE0LTczLjc3NGMyOC40NDctNDguNzg2IDEwLjA2MS0xMDQuMzQ3LTQuNy0xNDAuNDc5Yy00Ni42MDIgMzUuNzk5LTEzNy4yODkgOTguMzU3LTE3Ni4zMDQgMTU0LjQ3N2MtMjkuNzEgNDIuNzM0LTIuMzU4IDE0MS45OTcgNi45NzUgMTUxLjQ1NWM5LjMzMiA5LjQ1NyA4Mi43MDctNTMuNjk0IDgyLjcwNy01My42OTRjLTI3LjE0NyA3LjM1Mi01MC41OCAxMC41OTQtNjEuNTY5LjA3Nk02NC43MzggMTMwLjU0Yy41NzggOC4yMzMtNC45MTUgMjAuNTYtMjAuNTg0IDIxLjg2NWMtMTUuNjY4IDEuMzA2LTI1LjI1LTEzLjU5NC0yNS4zNS0xNy45NjVjLS4yODgtMTIuNzExIDkuMzktMTguMDU2IDIxLjA5LTE4LjA1NnMyNC4yNjYgNS45MjMgMjQuODQ0IDE0LjE1Nm0tOS42NCA2LjI4Yy43OTQtOC45NDItMy4zNzgtMTIuNTE4LTEzLjkwOC0xMi45MTVjLTEwLjUzLS4zOTgtMTMuOTA4IDcuMzUxLTEwLjczIDE0LjljMy4xOCA3LjU1IDIzLjg0MiA2Ljk1NSAyNC42MzctMS45ODZtNDguMDMzLTE1LjIwNWMxLjQzOS03LjQzNSA1Ljc2NC0xOS42NTYgMjkuOTc3LTIwLjYyNWMyOS45NzctMS4xOTkgMzQuMTg0IDIxLjY3MiAyOS43MzcgMzIuMzc2Yy02LjQ3NSAxNS41ODgtMjcuODE4IDI2LjM4LTQzLjQwNyAxNS4xMDhzLTE3Ljc0Ni0xOS40MjUtMTYuMzA3LTI2Ljg2bTEzLjY3IDcuOTE0czUuNTE1IDExLjc1IDE4LjIyNiAxMC41NTJjMTIuNzEtMS4yIDE2LjMwNy0xMC43OTIgMTYuNzg3LTE2LjA2OGMuMjE1LTIuMzczLTEuNjMxLTguNjg2LTYuNDc1LTExLjUxMWMtNS45MjgtMy40NTctMTUuNTM3LTMuMDcxLTIxLjM0NC0uOTZjLTEwLjU1MiAzLjgzNy03LjE5NSAxNy45ODctNy4xOTUgMTcuOTg3Ii8+Cjwvc3ZnPg==", "properties": { "ontology_graph_iri": { @@ -9917,7 +11646,7 @@ ], "main_category": "File", "description": "Read from or write to a JSON or JSON Lines file.", - "markdownDocumentation": "Typically, this dataset is used to transform an JSON file to another format, e.g., to RDF.\n\n## Reading\n\nIn addition to plain JSON files, *JSON Lines* files can also be read.\n\nFor reading, the JSON dataset supports a number of special paths:\n- `#id` is a special syntax for generating a hash-based id for a selected element. It can be used in URI patterns for entities which do not provide an identifier. Examples: `http://example.org/{#id}` or `http://example.org/{/pathToEntity/#id}`. If no URI pattern is configured, the dataset instead generates the default entity URI from the JSON source location.\n- `#text` retrieves the text of the selected node.\n- The backslash can be used to navigate to the parent JSON node, e.g., `\\parent/key`. The name of the backslash key (here `parent`) is ignored.\n\n## Writing\n\nWhen writing JSON, all entities need to possess a unique URI. Writing multiple root entities with the same URI will result in multiple entries in the generated JSON. If multiple nested entities with the same URI are written, only the last entity with a given URI will be written.\n", + "markdownDocumentation": "Typically, this dataset is used to transform an JSON file to another format, e.g., to RDF.\n\n## Reading\n\nIn addition to plain JSON files, *JSON Lines* files can also be read.\n\nFor reading, the JSON dataset supports a number of special paths:\n- `#id` is a special syntax for generating a hash-based id for a selected element. It can be used in URI patterns for entities which do not provide an identifier. Examples: `http://example.org/{#id}` or `http://example.org/{/pathToEntity/#id}`. If no URI pattern is configured, the dataset instead generates the default entity URI from the JSON source location.\n- `#text` retrieves the text of the selected node.\n- The backslash can be used to navigate to the parent JSON node, e.g., `\\parent/key`. The name of the backslash key (here `parent`) is ignored.\n\nKeys that are an empty string (`\"\": ...`) are ignored silently, without a warning or error: they are not offered as paths and cannot be addressed by a path. Their values are still included in `#text` and matched by `*`.\n\n## Writing\n\nWhen writing JSON, all entities need to possess a unique URI. Writing multiple root entities with the same URI will result in multiple entries in the generated JSON. If multiple nested entities with the same URI are written, only the last entity with a given URI will be written.\n", "pluginIcon": null, "properties": { "file": { @@ -10377,7 +12106,7 @@ ], "main_category": "Remote", "description": "Neo4j graph", - "markdownDocumentation": "\nSupports reading and writing Neo4j graphs. The following sections outline how graphs are generated and read back.\n\nFor more information about Neo4j, please refer to the [Neo4j documentation](https://neo4j.com/docs/).\n\n### Nodes\n\nFor each entity that is written to a Neo4j dataset, a _node_ will be created.\nA property `uri` will be added to each generated node, which holds the URI of the original entity.\nIn applications, the URI property should be used instead of the node identifiers, which are auto-generated in Neo4j and do not represent stable URIs.\n\nWhen reading nodes, the entity URIs will be generated based on that property.\nNodes that do not provide a `uri` property are read with a generated entity URI, which is not stable across reads.\n\n### Labels\n\n_Labels_ in Neo4j are used to group nodes into sets where all nodes that have a certain _label_ belongs to the same set.\nNeo4j _labels_ are comparable with _classes_ in RDF (not to be confused with labels in RDF).\n\nWhen writing entities to the Neo4j dataset, the following _labels_ will be added to each generated node:\n\n- For each entity _type_ (such as the _type_ set in a mapping), a _label_ will be added to the node in Neo4j.\n Since _types_ in eccenca DataIntegration are usually URIs, they will be converted according to the rules further down.\n- The _label_ as configured by the _label_ parameter on the Neo4j dataset itself.\n This is typically used to identify all entities that have been written by a certain Neo4j dataset specification in the project.\n For instance, if two Neo4j dataset specifications are added to a project - both writing to the same Neo4j database - different labels can be set to distinguish both sets of entities.\n In that respect it may be used to model a similar concept as _graphs_ in RDF.\n\n### Relationships\n\nA relationship connects two nodes in Neo4j.\nHierarchical mappings will generate relationships for all object mappings.\n\nRelationships can be addressed with property paths in mappings.\nAt the moment, only paths of length 1 are supported, i.e., it's not possible to use non-property paths.\n\n### Handling of URIs\n\nIn eccenca DataIntegration, URIs are typically used to uniquely identify classes and properties.\nWhile URIs are central in RDF, Neo4j does allow arbitrary names and does not have any special support for URIs.\n\nWhen generating Neo4j labels, properties and relationships, URIs will be shortened according to the following rules.\n- If a registered project prefix matches a URI, a name `{prefixName}_{localPart}` will be generated. For instance, `http://xmlns.com/foaf/0.1/name` will become `foaf_name`.\n Note that underscores (`_`) are used instead of colons (`:`) to separate the namespace and the local name.\n The reason is that colons are reserved in the Cypher query language and some tools don't escape properly and fail on databases that use colons in names.\n- If no project prefix matches a URI, the URI will be used verbatim. This will look ugly in Neo4j tools, so generally it's recommended to define prefixes for all used namespaces.\n\nWhen reading generated entities, the URIs of the classes and properties will be reconstructed based on the prefix table of the project. If the prefixes change between writing and reading, different URIs will be generated.\n\n### RDF vs. Neo4j terminology\n\nNeo4j uses a different terminology than RDF or description logic.\nFor users familiar with RDF, the following table shows the correspondent terms for some central concepts.\nThis is meant to help understanding and does not aim to provide a precise mapping as there are semantic differences between Neo4j and RDF.\n\n| RDF | Neo4j |\n| --- |--- |\n| resource | node |\n| class | label |\n| datatype property | property |\n| object property | relationship |\n| graph | Do not exist in Neo4j, but labels can be used to mimic graphs. |\n", + "markdownDocumentation": "\nSupports reading and writing Neo4j graphs. The following sections outline how graphs are generated and read back.\n\nFor more information about Neo4j, please refer to the [Neo4j documentation](https://neo4j.com/docs/).\n\n### Nodes\n\nFor each entity that is written to a Neo4j dataset, a _node_ will be created.\nA property `uri` will be added to each generated node, which holds the URI of the original entity.\nIn applications, the URI property should be used instead of the node identifiers, which are auto-generated in Neo4j and do not represent stable URIs.\n\nWhen reading nodes, the entity URIs will be generated based on that property.\nNodes that do not provide a `uri` property are read with a generated entity URI, which is not stable across reads.\n\n### Labels\n\n_Labels_ in Neo4j are used to group nodes into sets where all nodes that have a certain _label_ belongs to the same set.\nNeo4j _labels_ are comparable with _classes_ in RDF (not to be confused with labels in RDF).\n\nWhen writing entities to the Neo4j dataset, the following _labels_ will be added to each generated node:\n\n- For each `rdf:type` value of the entity, a _label_ will be added to the node in Neo4j.\n This covers all target entity types set in a mapping as well as mapping rules with the target property `rdf:type`, which allow to set the labels per entity, for instance from the classes of the source resources.\n Since _types_ in eccenca DataIntegration are usually URIs, they will be converted according to the rules further down.\n- The _label_ as configured by the _label_ parameter on the Neo4j dataset itself.\n This is typically used to identify all entities that have been written by a certain Neo4j dataset specification in the project.\n For instance, if two Neo4j dataset specifications are added to a project - both writing to the same Neo4j database - different labels can be set to distinguish both sets of entities.\n In that respect it may be used to model a similar concept as _graphs_ in RDF.\n\nLabels are only added, never removed.\nIf the types of an entity change between two runs, clear the dataset before writing, otherwise the node keeps the labels of the previous run as well.\n\n### Relationships\n\nA relationship connects two nodes in Neo4j.\nHierarchical mappings will generate relationships for all object mappings.\n\nRelationships can be addressed with property paths in mappings.\nAt the moment, only paths of length 1 are supported, i.e., it's not possible to use non-property paths.\n\n### Handling of URIs\n\nIn eccenca DataIntegration, URIs are typically used to uniquely identify classes and properties.\nWhile URIs are central in RDF, Neo4j does allow arbitrary names and does not have any special support for URIs.\n\nWhen generating Neo4j labels, properties and relationships, URIs will be shortened according to the following rules.\n- If a registered project prefix matches a URI, a name `{prefixName}_{localPart}` will be generated. For instance, `http://xmlns.com/foaf/0.1/name` will become `foaf_name`.\n Note that underscores (`_`) are used instead of colons (`:`) to separate the namespace and the local name.\n The reason is that colons are reserved in the Cypher query language and some tools don't escape properly and fail on databases that use colons in names.\n- If no project prefix matches a URI, the URI will be used verbatim. This will look ugly in Neo4j tools, so generally it's recommended to define prefixes for all used namespaces.\n\nWhen reading generated entities, the URIs of the classes and properties will be reconstructed based on the prefix table of the project. If the prefixes change between writing and reading, different URIs will be generated.\n\n### RDF vs. Neo4j terminology\n\nNeo4j uses a different terminology than RDF or description logic.\nFor users familiar with RDF, the following table shows the correspondent terms for some central concepts.\nThis is meant to help understanding and does not aim to provide a precise mapping as there are semantic differences between Neo4j and RDF.\n\n| RDF | Neo4j |\n| --- |--- |\n| resource | node |\n| class | label |\n| datatype property | property |\n| object property | relationship |\n| graph | Do not exist in Neo4j, but labels can be used to mimic graphs. |\n", "pluginIcon": null, "properties": { "uri": { @@ -10663,7 +12392,7 @@ ], "main_category": "File", "description": "Dataset which retrieves and writes all entities from/to an RDF file. For reading, the dataset is loaded in-memory and thus the size is restricted by the available memory. Large datasets should be loaded into an external RDF store and retrieved using the SPARQL dataset instead.", - "markdownDocumentation": "## 1. Purpose\n\nThe **RDF file** dataset plugin reads RDF data from a local file (or ZIP archive) into the project and, for supported formats, can also write RDF data back to a file.\n\nTypical use cases:\n\n- Loading small to medium RDF files (e.g. `.ttl`, `.nt`, `.rdf`, `.nq`) into a project.\n- Providing a local RDF snapshot for exploration, mapping, or linking.\n- Exporting RDF data from a workflow into a simple N-Triples file.\n\n> **Important:** The entire dataset is loaded **in memory**. Very large files should be imported into an external RDF store and accessed via a SPARQL dataset instead.\n\n## 2. Input and output\n\n### Input\n\n- **Source:**\n - A single RDF file, or\n - A ZIP archive containing multiple RDF files.\n\nThe file is read from the configured resource; no upstream operators are required.\n\n### Output\n\n- **Data:** RDF graph(s) exposed as a dataset that can be:\n - queried (e.g. via SPARQL-based components),\n - used as input for transformations,\n - or used in linking tasks.\n- **Graphs:**\n - If a graph name is configured and supported by the chosen format, only that named graph is exposed.\n - Otherwise, the default graph is used.\n\n## 3. Configuration notes\n\nThe detailed parameter list is generated in the UI. This section explains how to think about the most relevant settings.\n\n### 3.1 File\n\n- Points to the RDF file or ZIP archive to be used as the dataset.\n- If the file is a ZIP archive:\n - all contained files are scanned,\n - only files matching the **ZIP file regex** (see below) are loaded.\n\n### 3.2 Format\n\n- Optional parameter that specifies the RDF serialization format (e.g. `RDF/XML`, `N-Triples`, `N-Quads`, `Turtle`).\n- If left empty:\n - the format is **auto-detected** from the file name/extension,\n - if no format can be detected, an error is raised and the dataset will not load.\n- Reading:\n - Supported formats include at least `RDF/XML`, `N-Triples`, `N-Quads`, and `Turtle`.\n- Writing:\n - **Only `N-Triples` is supported for writing.**\n - For other formats, the dataset is effectively read-only.\n\n### 3.3 Graph\n\n- Optional graph name (IRI) that tells the dataset which graph to expose.\n- Behaviour:\n - For **graph-aware formats** (e.g. `N-Quads`):\n - you must provide a graph name to select which graph is used.\n - For **graph-less formats** (e.g. `N-Triples`, `Turtle`):\n - the graph parameter is ignored and the data is treated as a single default graph.\n- If left empty:\n - the default graph of the loaded dataset is used.\n\n### 3.4 Entity list (advanced)\n\n- Advanced parameter that allows restricting which entities (subjects) are retrieved.\n- You can provide a list of entity IRIs, separated by whitespace or line breaks.\n- Typical use:\n - limit retrieval to a known set of resources during exploration,\n - avoid scanning the entire file when only specific entities are needed.\n\n### 3.5 ZIP file regex (advanced)\n\n- Only relevant if the configured file is a ZIP archive.\n- Regular expression that filters which files inside the ZIP are considered.\n - Default: `.*` (all files).\n- Examples:\n - `.*\\.ttl` \u2192 only Turtle files.\n - `data-.*\\.nt` \u2192 only N-Triples files whose names start with `data-`.\n\n## 4. Behaviour\n\nWhen the dataset is used (e.g. queried or read by downstream components), it behaves as follows:\n\n1. **File size check**\n - Before loading, the file size is checked against an internal maximum for in-memory processing.\n - If the file is too large, the dataset fails with a \u201cresource too large\u201d error rather than attempting to load it.\n\n2. **Loading RDF data**\n - The RDF file (or all matching files inside a ZIP) is parsed into an internal RDF dataset held in memory.\n - The dataset may contain:\n - a default graph,\n - and, depending on the format, one or more named graphs.\n\n3. **Graph selection**\n - If a graph name is configured and the selected format supports named graphs, that graph is used.\n - Otherwise, the default graph is used.\n - For graph-less formats (e.g. `N-Triples`, `Turtle`), the graph setting is ignored.\n\n4. **Data access**\n - Once loaded, the data can be:\n - queried for types and paths,\n - used to retrieve entities by schema or by URI,\n - sampled for values and schema extraction.\n - Repeated reads re-use the in-memory dataset as long as the underlying file has not changed.\n\n5. **File changes**\n - The dataset monitors changes to the underlying file:\n - if the file\u2019s modification timestamp changes,\n - the RDF data is reloaded the next time it is accessed.\n\n6. **Writing**\n - When used as a sink (writing RDF):\n - data is serialized as **N-Triples** to the configured file.\n - other output formats are not supported.\n\n## 5. Limitations and recommendations\n\n- **In-memory dataset**\n - All data is held in memory; this is suitable for:\n - small to medium RDF files,\n - test datasets,\n - local snapshots.\n - For large production datasets, prefer:\n - importing the data into an external RDF store,\n - and using a SPARQL dataset instead.\n\n- **Format restrictions**\n - Reading: multiple standard RDF formats are supported.\n - Writing: only `N-Triples` is available as output format.\n\n- **Graph handling**\n - Graph selection only works where the underlying format supports named graphs.\n - For formats without named graphs, the graph parameter is ignored.\n\n## 6. Examples\n\n### 6.1 Simple Turtle file as dataset\n\n- **File:** `data/example.ttl`\n- **Format:** (empty \u2192 auto-detects Turtle)\n- **Graph:** (empty \u2192 default graph)\n\nUse case:\n- Small RDF dataset loaded from a Turtle file for exploration and transformations.\n\n### 6.2 N-Quads file with explicit graph\n\n- **File:** `data/dump.nq`\n- **Format:** `N-Quads`\n- **Graph:** `http://example.com/graph/main`\n\nUse case:\n- Large export containing multiple graphs, but only a specific named graph is used in the project.\n\n### 6.3 ZIP archive with multiple RDF files\n\n- **File:** `snapshots/rdf-data.zip`\n- **Format:** (empty \u2192 formats are detected per file)\n- **ZIP file regex:** `.*\\.ttl`\n\nUse case:\n- ZIP archive containing various files; only Turtle files are loaded into the dataset.\n\n## 7. When to use this dataset\n\nUse the **RDF file** dataset when:\n\n- You have a **local RDF file or ZIP archive** you want to work with directly.\n- The data volume fits comfortably into memory.\n- You need a simple, file-based way to:\n - explore RDF data,\n - run mappings or link specifications,\n - or export RDF as N-Triples.\n\nFor large or frequently updated datasets, prefer an external RDF store and a **SPARQL** dataset.\n", + "markdownDocumentation": "## 1. Purpose\n\nThe **RDF file** dataset plugin reads RDF data from a local file (or ZIP archive) into the project and, for supported formats, can also write RDF data back to a file.\n\nTypical use cases:\n\n- Loading small to medium RDF files (e.g. `.ttl`, `.nt`, `.rdf`, `.nq`, `.jsonld`) into a project.\n- Providing a local RDF snapshot for exploration, mapping, or linking.\n- Exporting RDF data from a workflow into a simple N-Triples file.\n\n> **Important:** The entire dataset is loaded **in memory**. Very large files should be imported into an external RDF store and accessed via a SPARQL dataset instead.\n\n## 2. Input and output\n\n### Input\n\n- **Source:**\n - A single RDF file, or\n - A ZIP archive containing multiple RDF files.\n\nThe file is read from the configured resource; no upstream operators are required.\n\n### Output\n\n- **Data:** RDF graph(s) exposed as a dataset that can be:\n - queried (e.g. via SPARQL-based components),\n - used as input for transformations,\n - or used in linking tasks.\n- **Graphs:**\n - If a graph name is configured and supported by the chosen format, only that named graph is exposed.\n - Otherwise, the default graph is used.\n\n## 3. Configuration notes\n\nThe detailed parameter list is generated in the UI. This section explains how to think about the most relevant settings.\n\n### 3.1 File\n\n- Points to the RDF file or ZIP archive to be used as the dataset.\n- If the file is a ZIP archive:\n - all contained files are scanned,\n - only files matching the **ZIP file regex** (see below) are loaded.\n\n### 3.2 Format\n\n- Optional parameter that specifies the RDF serialization format (e.g. `RDF/XML`, `N-Triples`, `N-Quads`, `Turtle`, `JSON-LD`).\n- If left empty:\n - the format is **auto-detected** from the file name/extension,\n - if no format can be detected, an error is raised and the dataset will not load.\n- Reading:\n - Supported formats include at least `RDF/XML`, `N-Triples`, `N-Quads`, `Turtle`, `RDF/JSON`, and `JSON-LD`.\n - `JSON-LD` files (`.jsonld`) are read as JSON-LD 1.1. A context that the file references by URL instead of embedding it is fetched when the file is loaded, so that URL must be reachable from the server.\n- Writing:\n - **Only `N-Triples` is supported for writing.**\n - For other formats, the dataset is effectively read-only.\n\n### 3.3 Graph\n\n- Optional graph name (IRI) that tells the dataset which graph to expose.\n- Behaviour:\n - For **graph-aware formats** (e.g. `N-Quads`, or `JSON-LD` files that contain named graphs):\n - you must provide a graph name to select which graph is used.\n - For **graph-less formats** (e.g. `N-Triples`, `Turtle`):\n - the graph parameter is ignored and the data is treated as a single default graph.\n- If left empty:\n - the default graph of the loaded dataset is used.\n\n### 3.4 Entity list (advanced)\n\n- Advanced parameter that allows restricting which entities (subjects) are retrieved.\n- You can provide a list of entity IRIs, separated by whitespace or line breaks.\n- Typical use:\n - limit retrieval to a known set of resources during exploration,\n - avoid scanning the entire file when only specific entities are needed.\n\n### 3.5 ZIP file regex (advanced)\n\n- Only relevant if the configured file is a ZIP archive.\n- Regular expression that filters which files inside the ZIP are considered.\n - Default: `.*` (all files).\n- Examples:\n - `.*\\.ttl` \u2192 only Turtle files.\n - `data-.*\\.nt` \u2192 only N-Triples files whose names start with `data-`.\n\n## 4. Behaviour\n\nWhen the dataset is used (e.g. queried or read by downstream components), it behaves as follows:\n\n1. **File size check**\n - Before loading, the file size is checked against an internal maximum for in-memory processing.\n - If the file is too large, the dataset fails with a \u201cresource too large\u201d error rather than attempting to load it.\n\n2. **Loading RDF data**\n - The RDF file (or all matching files inside a ZIP) is parsed into an internal RDF dataset held in memory.\n - The dataset may contain:\n - a default graph,\n - and, depending on the format, one or more named graphs.\n\n3. **Graph selection**\n - If a graph name is configured and the selected format supports named graphs, that graph is used.\n - Otherwise, the default graph is used.\n - For graph-less formats (e.g. `N-Triples`, `Turtle`), the graph setting is ignored.\n\n4. **Data access**\n - Once loaded, the data can be:\n - queried for types and paths,\n - used to retrieve entities by schema or by URI,\n - sampled for values and schema extraction.\n - Repeated reads re-use the in-memory dataset as long as the underlying file has not changed.\n\n5. **File changes**\n - The dataset monitors changes to the underlying file:\n - if the file\u2019s modification timestamp changes,\n - the RDF data is reloaded the next time it is accessed.\n\n6. **Writing**\n - When used as a sink (writing RDF):\n - data is serialized as **N-Triples** to the configured file.\n - other output formats are not supported.\n\n## 5. Limitations and recommendations\n\n- **In-memory dataset**\n - All data is held in memory; this is suitable for:\n - small to medium RDF files,\n - test datasets,\n - local snapshots.\n - For large production datasets, prefer:\n - importing the data into an external RDF store,\n - and using a SPARQL dataset instead.\n\n- **Format restrictions**\n - Reading: multiple standard RDF formats are supported.\n - Writing: only `N-Triples` is available as output format.\n\n- **Graph handling**\n - Graph selection only works where the underlying format supports named graphs.\n - For formats without named graphs, the graph parameter is ignored.\n\n## 6. Examples\n\n### 6.1 Simple Turtle file as dataset\n\n- **File:** `data/example.ttl`\n- **Format:** (empty \u2192 auto-detects Turtle)\n- **Graph:** (empty \u2192 default graph)\n\nUse case:\n- Small RDF dataset loaded from a Turtle file for exploration and transformations.\n\n### 6.2 N-Quads file with explicit graph\n\n- **File:** `data/dump.nq`\n- **Format:** `N-Quads`\n- **Graph:** `http://example.com/graph/main`\n\nUse case:\n- Large export containing multiple graphs, but only a specific named graph is used in the project.\n\n### 6.3 ZIP archive with multiple RDF files\n\n- **File:** `snapshots/rdf-data.zip`\n- **Format:** (empty \u2192 formats are detected per file)\n- **ZIP file regex:** `.*\\.ttl`\n\nUse case:\n- ZIP archive containing various files; only Turtle files are loaded into the dataset.\n\n## 7. When to use this dataset\n\nUse the **RDF file** dataset when:\n\n- You have a **local RDF file or ZIP archive** you want to work with directly.\n- The data volume fits comfortably into memory.\n- You need a simple, file-based way to:\n - explore RDF data,\n - run mappings or link specifications,\n - or export RDF as N-Triples.\n\nFor large or frequently updated datasets, prefer an external RDF store and a **SPARQL** dataset.\n", "pluginIcon": null, "properties": { "file": { @@ -10837,7 +12566,7 @@ "writeStrategy": { "name": "writeStrategy", "title": "Write strategy", - "description": "If this dataset is written to, it can be selected if data is overwritten or appended.'", + "description": "How this dataset is written to. 'Create table' fails if the table already exists, 'Overwrite table' deletes an existing table before writing, and 'Append rows' adds records to an existing table, whose columns need to match the mapping. Creating and overwriting tables always needs the Spark modules. Appending rows needs them too, unless the database supports importing CSV files.", "type": "string", "parameterType": "enumeration", "value": "default", @@ -11048,7 +12777,8 @@ "parameterTypeDescription": "A character string.", "value": "-.snowflakecomputing.com", "advanced": false, - "visibleInDialog": true + "visibleInDialog": true, + "allowSensitiveVariables": false }, "port": { "name": "port", @@ -11059,7 +12789,8 @@ "parameterTypeDescription": "An integer number.", "value": "443", "advanced": false, - "visibleInDialog": true + "visibleInDialog": true, + "allowSensitiveVariables": false }, "user": { "name": "user", @@ -11070,7 +12801,8 @@ "parameterTypeDescription": "A character string.", "value": "", "advanced": false, - "visibleInDialog": true + "visibleInDialog": true, + "allowSensitiveVariables": false }, "password": { "name": "password", @@ -11081,7 +12813,8 @@ "parameterTypeDescription": "A password string.", "value": "", "advanced": false, - "visibleInDialog": true + "visibleInDialog": true, + "allowSensitiveVariables": false }, "privateKey": { "name": "privateKey", @@ -11092,7 +12825,8 @@ "parameterTypeDescription": "A password string.", "value": "", "advanced": false, - "visibleInDialog": true + "visibleInDialog": true, + "allowSensitiveVariables": false }, "privateKeyPassword": { "name": "privateKeyPassword", @@ -11103,7 +12837,8 @@ "parameterTypeDescription": "A password string.", "value": "", "advanced": false, - "visibleInDialog": true + "visibleInDialog": true, + "allowSensitiveVariables": false }, "additionalParameters": { "name": "additionalParameters", @@ -11114,7 +12849,8 @@ "parameterTypeDescription": "YAML key-value pairs", "value": "", "advanced": false, - "visibleInDialog": true + "visibleInDialog": true, + "allowSensitiveVariables": false }, "warehouse": { "name": "warehouse", @@ -11126,6 +12862,7 @@ "value": "", "advanced": false, "visibleInDialog": true, + "allowSensitiveVariables": false, "autoCompletion": { "allowOnlyAutoCompletedValues": false, "autoCompleteValueWithLabels": false, @@ -11150,6 +12887,7 @@ "value": "", "advanced": false, "visibleInDialog": true, + "allowSensitiveVariables": false, "autoCompletion": { "allowOnlyAutoCompletedValues": false, "autoCompleteValueWithLabels": false, @@ -11175,6 +12913,7 @@ "value": "", "advanced": false, "visibleInDialog": true, + "allowSensitiveVariables": false, "autoCompletion": { "allowOnlyAutoCompletedValues": false, "autoCompleteValueWithLabels": false, @@ -11201,6 +12940,7 @@ "value": "", "advanced": false, "visibleInDialog": true, + "allowSensitiveVariables": false, "autoCompletion": { "allowOnlyAutoCompletedValues": false, "autoCompleteValueWithLabels": false, @@ -11247,7 +12987,8 @@ "parameterType": "code-sql", "value": "", "advanced": false, - "visibleInDialog": true + "visibleInDialog": true, + "allowSensitiveVariables": false }, "groupBy": { "name": "groupBy", @@ -11258,7 +12999,8 @@ "parameterTypeDescription": "A character string.", "value": "", "advanced": false, - "visibleInDialog": true + "visibleInDialog": true, + "allowSensitiveVariables": false }, "orderBy": { "name": "orderBy", @@ -11269,7 +13011,8 @@ "parameterTypeDescription": "A character string.", "value": "", "advanced": false, - "visibleInDialog": true + "visibleInDialog": true, + "allowSensitiveVariables": false }, "limit": { "name": "limit", @@ -11280,7 +13023,8 @@ "parameterTypeDescription": "An optional integer number.", "value": "10", "advanced": false, - "visibleInDialog": true + "visibleInDialog": true, + "allowSensitiveVariables": false }, "queryStrategy": { "name": "queryStrategy", @@ -11292,6 +13036,7 @@ "value": "access-complete-table", "advanced": false, "visibleInDialog": true, + "allowSensitiveVariables": false, "autoCompletion": { "allowOnlyAutoCompletedValues": true, "autoCompleteValueWithLabels": true, @@ -11307,7 +13052,8 @@ "parameterTypeDescription": "A character string.", "value": "", "advanced": false, - "visibleInDialog": true + "visibleInDialog": true, + "allowSensitiveVariables": false } } }, @@ -11329,13 +13075,14 @@ "writeStrategy": { "name": "writeStrategy", "title": "Write strategy", - "description": "If this dataset is written to, it can be selected if data is overwritten or appended.", + "description": "How this dataset is written to. 'Create table' fails if the table already exists, 'Overwrite table' deletes an existing table before writing, and 'Append rows' adds records to an existing table, whose columns need to match the mapping. Creating and overwriting tables always needs the Spark modules. Appending rows needs them too, unless the database supports importing CSV files.", "type": "string", "parameterType": "enumeration", "parameterTypeDescription": "One of the following values: default, overwrite-table, update-table", "value": "default", "advanced": false, "visibleInDialog": true, + "allowSensitiveVariables": false, "autoCompletion": { "allowOnlyAutoCompletedValues": true, "autoCompleteValueWithLabels": true, @@ -11352,6 +13099,7 @@ "value": "concatenateValuesStrategy", "advanced": false, "visibleInDialog": true, + "allowSensitiveVariables": false, "autoCompletion": { "allowOnlyAutoCompletedValues": true, "autoCompleteValueWithLabels": true, @@ -11386,7 +13134,8 @@ "parameterTypeDescription": "An integer number.", "value": "0", "advanced": true, - "visibleInDialog": true + "visibleInDialog": true, + "allowSensitiveVariables": false }, "pause": { "name": "pause", @@ -11397,7 +13146,8 @@ "parameterTypeDescription": "An integer number.", "value": "2000", "advanced": true, - "visibleInDialog": true + "visibleInDialog": true, + "allowSensitiveVariables": false } } } @@ -13373,8 +15123,8 @@ "Value" ], "main_category": "Value", - "description": "Calculates a single hash value covering all input values combined, across all input ports. Values are fed into the hash function in port order without any separator between them.", - "markdownDocumentation": "The **Combined input hash** operator produces exactly one hash value covering all input values combined, across all connected input ports. However many values arrive and however many ports are connected, the output is always a single string.\n\n## How combining works\n\nAll values from all input ports are fed sequentially into a single hash function \u2014 port 1 first, then port 2, and so on. Within each port, values are processed in the order they arrive. No separator is inserted between values or between ports. The hash covers the concatenated byte content of all values in that traversal order.\n\nThis means the result depends on both the content and the order of values. The same set of values in a different order produces a different hash. Connecting one port with values `[\"apple\", \"banana\"]` produces the same hash as connecting two ports with `[\"apple\"]` and `[\"banana\"]` respectively, because the bytes are fed in the same sequence either way.\n\n## Output\n\nThe output is a single lowercase hexadecimal string. The length depends on the algorithm: 64 characters for SHA-256, 32 for MD5, 40 for SHA-1, 96 for SHA-384, 128 for SHA-512. If the input is empty, the output is the hash of an empty message.\n\nValues are encoded as UTF-8 before hashing.\n\n## Algorithm parameter\n\nThe algorithm parameter selects the hash function. The default is SHA-256. The following algorithms from the [SPARQL 1.1 specification](https://www.w3.org/TR/sparql11-query/#func-hash) are supported:\n\n| SPARQL name | Java name | Notes |\n|-------------|-----------|-------|\n| MD5 | MD5 | Weak \u2014 vulnerable to collision attacks. Avoid for security-sensitive use. |\n| SHA1 | SHA-1 | Weak \u2014 deprecated for most security purposes. |\n| SHA256 | SHA-256 | Recommended default. |\n| SHA384 | SHA-384 | Stronger than SHA-256. |\n| SHA512 | SHA-512 | Strongest in the SPARQL set. |\n\nAdditional algorithms available on the JVM (such as SHA-512/256 and SHA-3 variants) are also accepted. The full list is JVM-dependent and visible in the algorithm parameter dropdown.\n\nNote that the Java names use hyphens (SHA-256, SHA-1) where SPARQL uses none (SHA256, SHA1). Both forms are accepted by this operator.\n\n## Examples\n\n**Notation:** List of values are represented via square brackets. Example: `[first, second]` represents a list of two values \"first\" and \"second\".\n\n---\n**A single input value produces one combined SHA-256 hash:**\n\n* Input values:\n 1. `[input value]`\n\n* Returns: `[f708c2afff0ed197e8551c4dd549ee5b848e0b407106cbdb8e451c8cd1479362]`\n\n\n---\n**Multiple values on one input are combined into a single hash:**\n\n* Input values:\n 1. `[apple, banana]`\n\n* Returns: `[5b692305517af54eb5ae12b9ff89eaf89e31f6a6ee208365886a18b81a2fc2f8]`\n\n\n---\n**Reversing the value order produces a different hash, confirming order-sensitivity:**\n\n* Input values:\n 1. `[banana, apple]`\n\n* Returns: `[d4183362b538440bb9a5f82359791c647280e6b657a1812f16f7bcc2b8f141ca]`\n\n\n---\n**Values from multiple ports are combined in port order, producing the same hash as the equivalent single-port sequence:**\n\n* Input values:\n 1. `[apple]`\n 2. `[banana]`\n\n* Returns: `[5b692305517af54eb5ae12b9ff89eaf89e31f6a6ee208365886a18b81a2fc2f8]`\n\n\n---\n**The algorithm parameter selects the hash function (MD5):**\n\n* Parameters\n * algorithm: `MD5`\n\n* Input values:\n 1. `[input value]`\n\n* Returns: `[cee963a28f70ee97751a85ef732e66dd]`\n\n\n---\n**The algorithm parameter selects the hash function (SHA-1):**\n\n* Parameters\n * algorithm: `SHA-1`\n\n* Input values:\n 1. `[apple]`\n\n* Returns: `[d0be2dc421be4fcd0172e5afceea3970e2f3d940]`\n\n\n---\n**The algorithm parameter selects the hash function (SHA-384):**\n\n* Parameters\n * algorithm: `SHA-384`\n\n* Input values:\n 1. `[apple]`\n\n* Returns: `[3d8786fcb588c93348756c6429717dc6c374a14f7029362281a3b21dc10250ddf0d0578052749822eb08bc0dc1e68b0f]`\n\n\n---\n**The algorithm parameter selects the hash function (SHA-512):**\n\n* Parameters\n * algorithm: `SHA-512`\n\n* Input values:\n 1. `[apple]`\n\n* Returns: `[844d8779103b94c18f4aa4cc0c3b4474058580a991fba85d3ca698a0bc9e52c5940feb7a65a3a290e17e6b23ee943ecc4f73e7490327245b4fe5d5efb590feb2]`\n\n\n---\n**Empty input produces the hash of an empty message:**\n\n* Input values:\n 1. `[]`\n\n* Returns: `[e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855]`\n\n\n---\n**Empty algorithm string causes IllegalArgumentException:**\n\n* Parameters\n * algorithm: ``\n\n* Input values:\n 1. `[foo]`\n\n* Returns: `[]`\n* **Throws error:** `IllegalArgumentException`\n\n\n", + "description": "Calculates a single hash value covering all input values combined, across all input ports. Values are fed into the hash function in port order, with an optional separator (the glue parameter) inserted between them.", + "markdownDocumentation": "The **Combined input hash** operator produces exactly one hash value covering all input values combined, across all connected input ports, with an optional separator inserted between them.\n\n## How combining works\n\nAll values from all input ports are fed sequentially into a single hash function, in port order; within each port, values are processed in the order they arrive. An optional separator, set with the glue parameter, can be inserted between every adjacent pair of values in that order \u2014 see the Glue parameter section below. The hash covers the concatenated byte content of all values in that order, together with any inserted separator.\n\nSwapping two values, or changing a single character within one value, produces a different hash. Connecting one port with values `[\"apple\", \"banana\"]` produces the same hash as connecting two ports with `[\"apple\"]` and `[\"banana\"]` respectively, because the bytes are fed in the same sequence either way.\n\n## Output\n\nThe output is a single lowercase hexadecimal string whose length is fixed by the chosen algorithm: 64 characters for SHA-256, 32 for MD5, 40 for SHA-1, 96 for SHA-384, 128 for SHA-512. Connecting no values at all still produces a hash \u2014 the hash of the empty message, not an error.\n\nValues are encoded as UTF-8 before hashing, so the same text produces the same bytes regardless of platform or locale.\n\n## Algorithm parameter\n\nThe algorithm parameter selects the hash function. The following algorithms from the [SPARQL 1.1 specification](https://www.w3.org/TR/sparql11-query/#func-hash) are supported:\n\n| SPARQL name | Java name | Notes |\n|-------------|-----------|-------|\n| MD5 | MD5 | Has known collision vulnerabilities. |\n| SHA1 | SHA-1 | Deprecated for most security purposes. |\n| SHA256 | SHA-256 | Default. |\n| SHA384 | SHA-384 | |\n| SHA512 | SHA-512 | |\n\nAdditional algorithms available on the JVM (such as SHA-512/256 and SHA-3 variants) are also accepted. The full list is JVM-dependent and visible in the algorithm parameter dropdown.\n\nThe Java names use hyphens (SHA-256, SHA-1) where SPARQL uses none (SHA256, SHA1); both forms are accepted by this operator.\n\n## Glue parameter\n\nThe glue parameter sets a separator string that is inserted between every adjacent pair of values in the traversal order described above, defaulting to an empty string. Its text can contain the escaped sequences `\\n`, `\\t`, and `\\\\`, which are converted to a newline, a tab, and a backslash respectively before hashing.\n\nConnecting one port with values `[\"apple\", \"banana\"]` and a glue of `-` produces the same hash as connecting two ports with `[\"apple\"]` and `[\"banana\"]` respectively, using the same glue, because the glue is inserted at the same point in the byte sequence either way.\n\nAn input port without a value is skipped, and the glue is only inserted between the values that exist. To insert the glue for such a port, the port can be connected through a Default Value operator with an empty value.\n\n## Examples\n\n**Notation:** List of values are represented via square brackets. Example: `[first, second]` represents a list of two values \"first\" and \"second\".\n\n---\n**A single input value produces one combined SHA-256 hash:**\n\n* Input values:\n 1. `[input value]`\n\n* Returns: `[f708c2afff0ed197e8551c4dd549ee5b848e0b407106cbdb8e451c8cd1479362]`\n\n\n---\n**Multiple values on one input are combined into a single hash:**\n\n* Input values:\n 1. `[apple, banana]`\n\n* Returns: `[5b692305517af54eb5ae12b9ff89eaf89e31f6a6ee208365886a18b81a2fc2f8]`\n\n\n---\n**The order in which values are combined affects the resulting hash:**\n\n* Input values:\n 1. `[banana, apple]`\n\n* Returns: `[d4183362b538440bb9a5f82359791c647280e6b657a1812f16f7bcc2b8f141ca]`\n\n\n---\n**Values from multiple ports are combined in port order: port 1's values first, then port 2's:**\n\n* Input values:\n 1. `[apple]`\n 2. `[banana]`\n\n* Returns: `[5b692305517af54eb5ae12b9ff89eaf89e31f6a6ee208365886a18b81a2fc2f8]`\n\n\n---\n**The glue parameter inserts a separator between values:**\n\n* Parameters\n * glue: `-`\n\n* Input values:\n 1. `[apple, banana]`\n\n* Returns: `[d6300f978b8699862450cbabc35349239c09e95c356723c32fc1407ec8104c78]`\n\n\n---\n**Values from multiple ports are combined using the glue separator in port order: port 1's values first, then port 2's:**\n\n* Parameters\n * glue: `-`\n\n* Input values:\n 1. `[apple]`\n 2. `[banana]`\n\n* Returns: `[d6300f978b8699862450cbabc35349239c09e95c356723c32fc1407ec8104c78]`\n\n\n---\n**The glue parameter accepts escaped characters, here a newline:**\n\n* Parameters\n * glue: `\\n`\n\n* Input values:\n 1. `[apple, banana]`\n\n* Returns: `[5c573825e0a168e9e382f9b7e78e6603271634dceac792b9c92fa77a27d404f8]`\n\n\n---\n**Values containing non-ASCII characters are encoded as UTF-8 before being hashed:**\n\n* Parameters\n * glue: `-`\n\n* Input values:\n 1. `[caf\u00e9, banana]`\n\n* Returns: `[c17ffffc64b5ffd83dbc2a975134c65a7f7f457a898b6e1fc6d2db1217fa92ff]`\n\n\n---\n**The glue parameter accepts a separator longer than one character:**\n\n* Parameters\n * glue: `::`\n\n* Input values:\n 1. `[apple, banana]`\n\n* Returns: `[7f721cdbf970bad79a82e7d827e0e5eef3d8263610aa6f062427060f5d2379db]`\n\n\n---\n**The order of values combined with a glue separator affects the resulting hash:**\n\n* Parameters\n * glue: `-`\n\n* Input values:\n 1. `[banana, apple]`\n\n* Returns: `[4472ed905ebe09b0e7a37ffa5adc6b0b2f052a6d0045fc037e2e56cdb45fba0a]`\n\n\n---\n**The glue separator is never applied to a single value, since there is no adjacent value to separate it from:**\n\n* Parameters\n * glue: `-`\n\n* Input values:\n 1. `[solo]`\n\n* Returns: `[5364f2f2fc4f54e9d47ad29cfb08ef430c8153394bf2a0dff5cbe77a0ffef861]`\n\n\n---\n**An input port without a value is skipped, and no glue is inserted for it:**\n\n* Parameters\n * glue: `-`\n\n* Input values:\n 1. `[]`\n 2. `[solo]`\n\n* Returns: `[5364f2f2fc4f54e9d47ad29cfb08ef430c8153394bf2a0dff5cbe77a0ffef861]`\n\n\n---\n**An empty port 2 is skipped in the same way, producing the same hash as an empty port 1:**\n\n* Parameters\n * glue: `-`\n\n* Input values:\n 1. `[solo]`\n 2. `[]`\n\n* Returns: `[5364f2f2fc4f54e9d47ad29cfb08ef430c8153394bf2a0dff5cbe77a0ffef861]`\n\n\n---\n**Without any input values, the glue is not hashed on its own, producing the hash of an empty message:**\n\n* Parameters\n * glue: `-`\n\n* Input values:\n 1. `[]`\n 2. `[]`\n\n* Returns: `[e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855]`\n\n\n---\n**With more than two values, the glue separator is inserted between every adjacent pair:**\n\n* Parameters\n * glue: `-`\n\n* Input values:\n 1. `[a, b]`\n 2. `[c]`\n\n* Returns: `[cbd2be7b96f770a0326948ebd158cf539fab0627e8adbddc97f7a65c6a8ae59a]`\n\n\n---\n**Three values in one port produce a hash with the glue separator inserted between every pair:**\n\n* Parameters\n * glue: `-`\n\n* Input values:\n 1. `[a, b, c]`\n\n* Returns: `[cbd2be7b96f770a0326948ebd158cf539fab0627e8adbddc97f7a65c6a8ae59a]`\n\n\n---\n**A value that is identical to the glue string is hashed as ordinary byte content, with no special escaping:**\n\n* Parameters\n * glue: `-`\n\n* Input values:\n 1. `[a, -, b]`\n\n* Returns: `[1acd27f89b4f44ddaf3b81fadb5491ea61f9ae3981545cf62d3502a387bcf350]`\n\n\n---\n**Values from three ports are combined using the glue separator in port order: port 1's value, then port 2's, then port 3's:**\n\n* Parameters\n * glue: `-`\n\n* Input values:\n 1. `[a]`\n 2. `[b]`\n 3. `[c]`\n\n* Returns: `[cbd2be7b96f770a0326948ebd158cf539fab0627e8adbddc97f7a65c6a8ae59a]`\n\n\n---\n**The order in which values are combined affects the resulting hash, even with more than two values and a glue separator:**\n\n* Parameters\n * glue: `-`\n\n* Input values:\n 1. `[c, b]`\n 2. `[a]`\n\n* Returns: `[8e599734c8f75bd7866757e5f9cd4ab3cc173d753fb858a85ae6efb817e3e200]`\n\n\n---\n**With more than two values, the SHA-1 algorithm still inserts the glue separator between every adjacent pair:**\n\n* Parameters\n * algorithm: `SHA-1`\n * glue: `-`\n\n* Input values:\n 1. `[a, b]`\n 2. `[c]`\n\n* Returns: `[e088d9e3f737c091378fe8494936b16d51eb42ee]`\n\n\n---\n**With the SHA-512 algorithm selected, the glue separator is inserted between every adjacent pair of three values:**\n\n* Parameters\n * algorithm: `SHA-512`\n * glue: `-`\n\n* Input values:\n 1. `[a, b]`\n 2. `[c]`\n\n* Returns: `[a271a54e93179f7dccb132a17c914e66e76a91ae3b7cb8b3abffa21a8ffb66b65a2883769bc4c6f95ab39fbd039cfe21046b485cef1588269d2605c7253f2b98]`\n\n\n---\n**With the MD5 algorithm selected, the glue parameter still inserts a separator between values:**\n\n* Parameters\n * algorithm: `MD5`\n * glue: `-`\n\n* Input values:\n 1. `[apple, banana]`\n\n* Returns: `[9b22c402a5166c71ebdf2d1b99f37727]`\n\n\n---\n**With the MD5 algorithm selected, escaped glue characters are still converted before being used as the separator:**\n\n* Parameters\n * algorithm: `MD5`\n * glue: `\\n`\n\n* Input values:\n 1. `[apple, banana]`\n\n* Returns: `[cf0bf238cadd6688db6f8d2511a52ebf]`\n\n\n---\n**With the MD5 algorithm and a glue of '-', the hash equals the SPARQL expression MD5(CONCAT(?a, \"-\", ?b)):**\n\n* Parameters\n * algorithm: `MD5`\n * glue: `-`\n\n* Input values:\n 1. `[a]`\n 2. `[b]`\n\n* Returns: `[8ca2ed590cf2ea2404f2e67641bcdf50]`\n\n\n---\n**The algorithm parameter selects the hash function (MD5):**\n\n* Parameters\n * algorithm: `MD5`\n\n* Input values:\n 1. `[input value]`\n\n* Returns: `[cee963a28f70ee97751a85ef732e66dd]`\n\n\n---\n**The algorithm parameter selects the hash function (SHA-1):**\n\n* Parameters\n * algorithm: `SHA-1`\n\n* Input values:\n 1. `[apple]`\n\n* Returns: `[d0be2dc421be4fcd0172e5afceea3970e2f3d940]`\n\n\n---\n**The algorithm parameter selects the hash function (SHA-384):**\n\n* Parameters\n * algorithm: `SHA-384`\n\n* Input values:\n 1. `[apple]`\n\n* Returns: `[3d8786fcb588c93348756c6429717dc6c374a14f7029362281a3b21dc10250ddf0d0578052749822eb08bc0dc1e68b0f]`\n\n\n---\n**The algorithm parameter selects the hash function (SHA-512):**\n\n* Parameters\n * algorithm: `SHA-512`\n\n* Input values:\n 1. `[apple]`\n\n* Returns: `[844d8779103b94c18f4aa4cc0c3b4474058580a991fba85d3ca698a0bc9e52c5940feb7a65a3a290e17e6b23ee943ecc4f73e7490327245b4fe5d5efb590feb2]`\n\n\n---\n**Empty input produces the hash of an empty message:**\n\n* Input values:\n 1. `[]`\n\n* Returns: `[e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855]`\n\n\n---\n**Empty algorithm string causes IllegalArgumentException:**\n\n* Parameters\n * algorithm: ``\n\n* Input values:\n 1. `[foo]`\n\n* Returns: `[]`\n* **Throws error:** `IllegalArgumentException`\n\n\n", "pluginIcon": null, "properties": { "algorithm": { @@ -13387,6 +15137,17 @@ "advanced": false, "visibleInDialog": true, "properties": {} + }, + "glue": { + "name": "glue", + "title": "Glue", + "description": "Separator to be inserted between two concatenated strings. The text can contain escaped characters \\n, \\t and \\\\ that are replaced by a newline, tab or backslash respectively.", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": false, + "visibleInDialog": true, + "properties": {} } }, "properties_advanced": {}, @@ -13407,6 +15168,18 @@ { "id": "mapWithDefaultInput", "description": "One hash value is produced for the entire set of inputs by the Combined input hash plugin. The Map with default plugin instead keeps a value sequence and rewrites it position by position through the mapping, falling back to the second input where no mapping entry is found." + }, + { + "id": "concat", + "description": "Combined input hash always returns exactly one value, however many values or ports are connected. Concatenate instead produces every combination of connected values as a separate output, so more than one value on an input produces multiple outputs instead of one." + }, + { + "id": "zip", + "description": "Combined input hash accepts any number of ports and always returns exactly one value. Zip instead requires exactly two inputs, pairs their values by position, and returns one output per position rather than one combined result." + }, + { + "id": "concatMultiValues", + "description": "Concatenate multiple values combines everything received on one port into a single string, but keeps each port's result separate; more ports still return separate results, one per port. Combined input hash instead crosses every port boundary, always returning one hash value overall." } ] }, @@ -13546,6 +15319,10 @@ { "id": "concatMultiValues", "description": "Passing multiple values to a single input of Concatenate does not combine them \u2014 it multiplies the output. Concatenate multiple values is the plugin that collapses multiple values within an input into one string, producing exactly one result per input." + }, + { + "id": "inputHash", + "description": "Concatenate returns one string only when each input provides exactly one value; an input with more than one value instead returns the Cartesian product as separate results. Combined input hash always returns exactly one hash value, no matter how many values or ports are connected." } ] }, @@ -13597,6 +15374,10 @@ { "id": "concat", "description": "Concatenate multiple values collapses all values within each input into one string, preserving the boundary between inputs. Concatenate crosses that boundary \u2014 it takes one value from each input and produces all combinations, so the output grows with the number of inputs and values." + }, + { + "id": "inputHash", + "description": "Combined input hash always produces exactly one hash value, however many ports or values are connected. Concatenate multiple values instead combines everything on one port into a single string, so connecting more ports returns more separate strings instead of one." } ] }, @@ -13834,7 +15615,7 @@ "description": "Set date (e.g.YYYY-MM-DD) to convert currencies based on historic rates.", "type": "string", "parameterType": "string", - "value": "2026-09-02", + "value": "2026-11-11", "advanced": false, "visibleInDialog": true, "properties": {} @@ -14354,8 +16135,8 @@ "Template" ], "main_category": "Template", - "description": "Evaluates a template. Input values can be addressed using the variables 'input1', 'input2', etc. Global variables are available in the 'global' scope, e.g., 'global.myVar'.", - "markdownDocumentation": "Evaluates a template. Input values can be addressed using the variables 'input1', 'input2', etc. Global variables are available in the 'global' scope, e.g., 'global.myVar'.\n\n## Examples\n\n**Notation:** List of values are represented via square brackets. Example: `[first, second]` represents a list of two values \"first\" and \"second\".\n\n---\n**Substitutes each input variable by its value:**\n\n* Parameters\n * template: \n ```\n Hello {{input1}} {{input2}},\n \n How are you today?\n ```\n\n* Input values:\n 1. `[John]`\n 2. `[Doe]`\n\n* Returns: \n ```\n [Hello John Doe,\n \n How are you today?]\n ```\n\n\n---\n**Concatenates all values of a multi-valued input:**\n\n* Parameters\n * template: `Hello {{input1}}`\n\n* Input values:\n 1. `[A, B]`\n\n* Returns: `[Hello AB]`\n\n\n---\n**Supports iterating over the values of a multi-valued input:**\n\n* Parameters\n * template: `Hello {% for value in input1 %}{{value}}, {% endfor %}how are you doing?`\n\n* Input values:\n 1. `[Bob, Eve]`\n\n* Returns: `[Hello Bob, Eve, how are you doing?]`\n\n\n---\n**Supports method calls on input values:**\n\n* Parameters\n * template: `{{ input1.trim() }}`\n\n* Input values:\n 1. `[ John ]`\n\n* Returns: `[John]`\n\n\n---\n**Rejects unscoped variables that are no input variables:**\n\n* Parameters\n * template: `Hello {{badVariable}} {{input1}}`\n\n* Input values:\n 1. `[John]`\n\n* Returns: `[]`\n* **Throws error:** `ValidationException`\n\n\n---\n**Rejects wrongly numbered input variables, the numbering starts at input1 without leading zeros:**\n\n* Parameters\n * template: `Hello {{input01}}`\n\n* Returns: `[]`\n* **Throws error:** `ValidationException`\n\n\n---\n**Rejects scoped variables from unknown scopes, e.g. misspelled scope names:**\n\n* Parameters\n * template: `Hello {{projekt.myVar}}`\n\n* Returns: `[]`\n* **Throws error:** `ValidationException`\n\n\n---\n**Fails at execution time if a referenced input is not connected:**\n\n* Parameters\n * template: `Hello {{input1}}`\n\n* Returns: `[]`\n* **Throws error:** `UnboundVariablesException`\n\n\n", + "description": "Evaluates a template. Input values can be addressed using the variables 'input1', 'input2', etc. Global variables are available in the 'global' scope, e.g., 'global.myVar', except for sensitive ones.", + "markdownDocumentation": "Evaluates a template. Input values can be addressed using the variables 'input1', 'input2', etc. Global variables are available in the 'global' scope, e.g., 'global.myVar', except for sensitive ones.\n\n## Examples\n\n**Notation:** List of values are represented via square brackets. Example: `[first, second]` represents a list of two values \"first\" and \"second\".\n\n---\n**Substitutes each input variable by its value:**\n\n* Parameters\n * template: \n ```\n Hello {{input1}} {{input2}},\n \n How are you today?\n ```\n\n* Input values:\n 1. `[John]`\n 2. `[Doe]`\n\n* Returns: \n ```\n [Hello John Doe,\n \n How are you today?]\n ```\n\n\n---\n**Concatenates all values of a multi-valued input:**\n\n* Parameters\n * template: `Hello {{input1}}`\n\n* Input values:\n 1. `[A, B]`\n\n* Returns: `[Hello AB]`\n\n\n---\n**Supports iterating over the values of a multi-valued input:**\n\n* Parameters\n * template: `Hello {% for value in input1 %}{{value}}, {% endfor %}how are you doing?`\n\n* Input values:\n 1. `[Bob, Eve]`\n\n* Returns: `[Hello Bob, Eve, how are you doing?]`\n\n\n---\n**Supports method calls on input values:**\n\n* Parameters\n * template: `{{ input1.trim() }}`\n\n* Input values:\n 1. `[ John ]`\n\n* Returns: `[John]`\n\n\n---\n**Rejects unscoped variables that are no input variables:**\n\n* Parameters\n * template: `Hello {{badVariable}} {{input1}}`\n\n* Input values:\n 1. `[John]`\n\n* Returns: `[]`\n* **Throws error:** `ValidationException`\n\n\n---\n**Rejects wrongly numbered input variables, the numbering starts at input1 without leading zeros:**\n\n* Parameters\n * template: `Hello {{input01}}`\n\n* Returns: `[]`\n* **Throws error:** `ValidationException`\n\n\n---\n**Rejects scoped variables from unknown scopes, e.g. misspelled scope names:**\n\n* Parameters\n * template: `Hello {{projekt.myVar}}`\n\n* Returns: `[]`\n* **Throws error:** `ValidationException`\n\n\n---\n**Fails at execution time if a referenced input is not connected:**\n\n* Parameters\n * template: `Hello {{input1}}`\n\n* Returns: `[]`\n* **Throws error:** `UnboundVariablesException`\n\n\n", "pluginIcon": null, "properties": { "template": { @@ -15977,7 +17758,7 @@ "Replace" ], "main_category": "Replace", - "description": "Replaces values based on a map of values read from a file in Open XML format (XLSX). The XLSX file may contain several sheets of the form: ``` mapFrom,mapTo , ``` and more. An empty string can be created in Excel and alternatives by inserting `=\"\"` in the input line of a cell. If there are multiple values for a single key, all values will be returned for the given key. Note that the mapping table will be cached in memory. If the Excel file is updated (even while transforming), the map will be reloaded within seconds.", + "description": "Replaces values based on a map of values read from a file in Open XML format (XLSX). The XLSX file may contain several sheets of the form: `mapFrom,mapTo ,` and more. An empty string can be created in Excel and alternatives by inserting `=\"\"` in the input line of a cell. If there are multiple values for a single key, all values will be returned for the given key. Note that the mapping table will be cached in memory. If the Excel file is updated (even while transforming), the map will be reloaded within seconds.", "markdownDocumentation": "Replaces values based on a map of values read from a file in Open XML format (XLSX).\nThe XLSX file may contain several sheets of the form:\n```\nmapFrom,mapTo\n,\n```\nand more.\n\nAn empty string can be created in Excel and alternatives by inserting `=\"\"` in the input line of a cell.\n\nIf there are multiple values for a single key, all values will be returned for the given key.\n\nNote that the mapping table will be cached in memory. If the Excel file is updated (even while transforming), the map will be reloaded within seconds.\n \n", "pluginIcon": null, "properties": { @@ -18599,7 +20380,12 @@ "TransformOperator" ], "pluginType": "transformer", - "relatedPlugins": [] + "relatedPlugins": [ + { + "id": "filterByRegex", + "description": "Filter by regex tests each value against a pattern; the length is never a factor. Filter by length checks the length bounds only; a pattern is never a factor." + } + ] }, "filterByRegex": { "pluginId": "filterByRegex", @@ -18608,14 +20394,14 @@ "Filter" ], "main_category": "Filter", - "description": "Removes all strings that do NOT match a regex. If 'negate' is true, only strings will be removed that match the regex.", - "markdownDocumentation": "Removes all strings that do NOT match a regex. If 'negate' is true, only strings will be removed that match the regex.\n", + "description": "This transformer filters values by a regular expression: values matching the regex are kept, unless 'negate' is set, which keeps only the values that don't. By default the regex must match a value in full; setting 'contains' relaxes that so any value containing a match is kept instead.", + "markdownDocumentation": "## Description\n\nThe Filter by regex plugin keeps or drops values based on a regular expression. By default, a value is kept only if it matches the regex in full; setting `contains` relaxes that to keep any value the regex matches anywhere within. Setting `negate` inverts whichever of those two decisions is in effect.\n\n## Notes on regular expressions\n\nAttention: regex metacharacters in the pattern have to be escaped to be matched literally, e.g. a literal dot needs `\\.`, not `.`, which otherwise means \"any character.\"\n\nA pattern that can match an empty string, such as `a*`, matches every value under `contains`, since an empty match always \"occurs somewhere\" in any string.\n\nA regex that fails to compile throws a `PatternSyntaxException` rather than silently doing nothing.\n\nAttention: This happens during _execution_.\nIn other words: It's technically possible to save _invalid_ regular expressions.\n\n### Note for advanced users\n\nA compilation of the available constructs for building regular expressions is available in the\n[API of the Java `Pattern`](https://docs.oracle.com/en/java/javase/25/docs/api/java.base/java/util/regex/Pattern.html).\n\n## Missing or empty input\n\nIf no input is connected to this transformer, it throws a `ValidationException`. If an input is connected but carries no values, the transformer returns nothing for that entity \u2014 this is not an error.\n\n## Examples\n\n**Notation:** List of values are represented via square brackets. Example: `[first, second]` represents a list of two values \"first\" and \"second\".\n\n---\n**Keeps only values that fully match the regex:**\n\n* Parameters\n * regex: `abc`\n\n* Input values:\n 1. `[abc, xabcx, xyz]`\n\n* Returns: `[abc]`\n\n\n---\n**Keeps only values consisting of exactly three word characters:**\n\n* Parameters\n * regex: `\\w\\w\\w`\n\n* Input values:\n 1. `[abc, ab, abcd, a-x]`\n\n* Returns: `[abc]`\n\n\n---\n**Excludes values that fully match the regex, keeping the rest:**\n\n* Parameters\n * regex: `abc`\n * negate: `true`\n\n* Input values:\n 1. `[abc, xabcx, xyz]`\n\n* Returns: `[xabcx, xyz]`\n\n\n---\n**Keeps values that contain a match anywhere, not just full matches:**\n\n* Parameters\n * regex: `abc`\n * contains: `true`\n\n* Input values:\n 1. `[abc, xabcx, xyz]`\n\n* Returns: `[abc, xabcx]`\n\n\n---\n**Excludes values that contain a match anywhere, keeping the rest:**\n\n* Parameters\n * regex: `abc`\n * negate: `true`\n * contains: `true`\n\n* Input values:\n 1. `[abc, xabcx, xyz]`\n\n* Returns: `[xyz]`\n\n\n---\n**A pattern matching zero characters finds a match in any value, so 'contains' keeps everything:**\n\n* Parameters\n * regex: `a*`\n * contains: `true`\n\n* Input values:\n 1. `[xyz, aaa, ]`\n\n* Returns: `[xyz, aaa, ]`\n\n\n---\n**An anchored pattern still only matches at the anchored position, even with 'contains' set:**\n\n* Parameters\n * regex: `^abc`\n * contains: `true`\n\n* Input values:\n 1. `[abcxyz, xabcx, xyz]`\n\n* Returns: `[abcxyz]`\n\n\n---\n**Throws when the regex fails to compile:**\n\n* Parameters\n * regex: `(`\n\n* Input values:\n 1. `[abc]`\n\n* Returns: `[]`\n* **Throws error:** `PatternSyntaxException`\n\n\n---\n**A backslash-escaped metacharacter in the regex is matched literally:**\n\n* Parameters\n * regex: `a\\.b`\n\n* Input values:\n 1. `[a.b, aXb]`\n\n* Returns: `[a.b]`\n\n\n---\n**A whitespace-only value matches a whitespace pattern:**\n\n* Parameters\n * regex: `\\s+`\n\n* Input values:\n 1. `[ , abc]`\n\n* Returns: `[ ]`\n\n\n---\n**A Unicode value matches a literal pattern correctly:**\n\n* Parameters\n * regex: `caf\u00e9`\n\n* Input values:\n 1. `[caf\u00e9, cafe]`\n\n* Returns: `[caf\u00e9]`\n\n\n---\n**Returns nothing when the connected input carries no values:**\n\n* Parameters\n * regex: `abc`\n\n* Input values:\n 1. `[]`\n\n* Returns: `[]`\n\n\n---\n**Throws when no input is connected:**\n\n* Parameters\n * regex: `abc`\n\n* Returns: `[]`\n* **Throws error:** `ValidationException`\n\n\n", "pluginIcon": null, "properties": { "regex": { "name": "regex", "title": "Regex", - "description": "No description", + "description": "The regular expression to test each value against.", "type": "string", "parameterType": "string", "value": null, @@ -18626,7 +20412,18 @@ "negate": { "name": "negate", "title": "Negate", - "description": "No description", + "description": "If true, keeps values that do not match instead of values that do.", + "type": "string", + "parameterType": "boolean", + "value": "false", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "contains": { + "name": "contains", + "title": "Contains", + "description": "If true, the pattern only needs to occur somewhere in the value.", "type": "string", "parameterType": "boolean", "value": "false", @@ -18650,7 +20447,31 @@ "relatedPlugins": [ { "id": "regexSelect", - "description": "Filter by regex keeps or drops values from the input sequence based on full-string matching. Regex selection keeps the checked value out of the output and instead returns a pattern-list-shaped result filled with the provided output value where a pattern matches." + "description": "Filter by regex keeps or drops values from the input sequence based on regex matching. Regex selection keeps the checked value out of the output and instead returns a pattern-list-shaped result filled with the provided output value where a pattern matches." + }, + { + "id": "validateRegex", + "description": "Validate regex throws on the first non-matching value, failing the whole operation. Filter by regex just drops it and keeps the rest." + }, + { + "id": "regexReplace", + "description": "Filter by regex is a structural filter: it never changes a value's content, only whether that value appears in the output. Regex replace works the other way: it changes what's inside each value, but every value that goes in still comes out." + }, + { + "id": "regexExtract", + "description": "Regex extract never returns the original value \u2014 only part of the match itself, a substring or a capturing group. By contrast, Filter by regex returns only the original value, unmodified, or nothing at all." + }, + { + "id": "ifMatchesRegex", + "description": "If matches regex checks one sequence for a match, then returns one of two candidate sequences, completely unchanged. Each value in a single sequence is tested on its own under Filter by regex, kept or dropped to build the output." + }, + { + "id": "filterByLength", + "description": "Filter by length only checks length, bounded by a minimum and a maximum. Length is irrelevant to Filter by regex; only the pattern matters." + }, + { + "id": "removeValues", + "description": "Remove values checks each value against a fixed, comma-separated blacklist \u2014 an exact, case-insensitive match, nothing computed. Filter by regex checks each value against a regular expression instead, which can be a single word or a broad, open-ended pattern." } ] }, @@ -18750,14 +20571,14 @@ "Tokenization" ], "main_category": "Sequence", - "description": "Returns the value found at the specified index. Fails or returns an empty result depending on failIfNoFound is set or not. Please be aware that this will work only if the data source supports some kind of ordering like XML or JSON. This is probably not a good idea to do with RDF models. If emptyStringToEmptyResult is true then instead of a result with an empty String, an empty result is returned.", - "markdownDocumentation": "Returns the value found at the specified index. Fails or returns an empty result depending on failIfNoFound is set or not.\n Please be aware that this will work only if the data source supports some kind of ordering like XML or JSON. This\n is probably not a good idea to do with RDF models.\n\n If emptyStringToEmptyResult is true then instead of a result with an empty String, an empty result is returned.\n \n", + "description": "Returns the value found at the specified index. A negative index counts from the end of the sequence.", + "markdownDocumentation": "## Description\n\nThis transformer extracts a single value from a sequence at a fixed position \u2014 useful whenever a data source\nreturns its values in a known, meaningful order and only one of them is needed. A positive index counts from the\nstart of the sequence, `0`-based. A negative index counts from the end \u2014 `-1` is the last value, `-2` the\nsecond-to-last, and so on. An index equal to the negative length of the sequence wraps around to the first value,\nrather than falling out of range.\n\nIf no value exists at the given index, the behavior depends on `failIfNotFound`: by default the input is simply\ndropped from the result, but if `failIfNotFound` is enabled, the transformation fails instead.\n\nEach input value is a sequence in its own right, and the index is resolved independently for each one. If this\ntransformer receives several input values, every one of them is indexed on its own, not treated as a single\ncombined sequence.\n\nIndexing only makes sense where the data source preserves some kind of order, such as XML or JSON. RDF has no\ninherent ordering, so using this transformer on values sourced from an RDF model is not recommended \u2014 \"index 0\"\nagainst such a sequence would be arbitrary, and could change between runs.\n\nIf `emptyStringToEmptyResult` is enabled, a result that is an empty string is treated the same as no result at\nall, rather than being returned as an empty string.\n\n## Example usage\n\nGiven the input sequence `a, b, c`:\n\n- Index `-1` returns `c` \u2014 the last value.\n- Index `-3` \u2014 equal to the negative length of the sequence \u2014 wraps around and returns `a`, the first value.\n\nGiven the single-element input sequence `a`:\n\n- Index `-2` is past the start of the sequence, so by default the result is empty.\n- Index `-2` with `failIfNotFound` enabled throws an error instead of returning an empty result.\n\n## Examples\n\n**Notation:** List of values are represented via square brackets. Example: `[first, second]` represents a list of two values \"first\" and \"second\".\n\n---\n**A negative index counts from the end; -1 returns the last value:**\n\n* Parameters\n * index: `-1`\n\n* Input values:\n 1. `[a, b, c]`\n\n* Returns: `[c]`\n\n\n---\n**An index equal to the negative sequence length wraps to the first value:**\n\n* Parameters\n * index: `-3`\n\n* Input values:\n 1. `[a, b, c]`\n\n* Returns: `[a]`\n\n\n---\n**A negative index past the start of the sequence returns an empty result by default:**\n\n* Parameters\n * index: `-2`\n\n* Input values:\n 1. `[a]`\n\n* Returns: `[]`\n\n\n---\n**A negative index past the start of the sequence throws if 'failIfNotFound' is enabled:**\n\n* Parameters\n * index: `-2`\n * failIfNotFound: `true`\n\n* Input values:\n 1. `[a]`\n\n* Returns: `[]`\n* **Throws error:** `IndexOutOfBoundsException`\n\n\n", "pluginIcon": null, "properties": { "index": { "name": "index", "title": "Index", - "description": "No description", + "description": "The index of the value: 0-based if non-negative; if negative, counts from the end, wrapping at -length.", "type": "string", "parameterType": "int", "value": null, @@ -18768,7 +20589,7 @@ "failIfNotFound": { "name": "failIfNotFound", "title": "Fail if not found", - "description": "No description", + "description": "If enabled, the transformation fails when no value exists at the given index; otherwise it is dropped.", "type": "string", "parameterType": "boolean", "value": "false", @@ -18779,7 +20600,7 @@ "emptyStringToEmptyResult": { "name": "emptyStringToEmptyResult", "title": "Empty string to empty result", - "description": "No description", + "description": "If enabled, an empty-string result is treated as no result, instead of being returned as an empty string.", "type": "string", "parameterType": "boolean", "value": "false", @@ -18920,6 +20741,10 @@ { "id": "regexExtract", "description": "The Regex extract plugin returns the matching content from the input string, or the first capturing group if the regular expression contains capturing groups. The If matches regex plugin does not return matched content; it uses the match only to choose which of the provided input values becomes the output." + }, + { + "id": "filterByRegex", + "description": "Filter by regex looks at values one at a time, deciding for each whether it belongs in the output. If matches regex takes an entirely different approach: the whole first sequence is checked at once, and whichever of the two other sequences applies comes back exactly as given." } ] }, @@ -19001,7 +20826,7 @@ ], "main_category": "Uncategorized", "description": "Process a JSON path with a jq filter / program.", - "markdownDocumentation": "\n> [jq](https://jqlang.org/) is like sed for JSON data - you can use it to\n> slice and filter and map and transform structured data with the same ease that sed, awk,\n> grep and friends let you play with text.\n\nIn order to test jq expressions, you can use [play.jqlang.org](https://play.jqlang.org/).\n\n## Basic concepts:\n\n- Filters separated by a comma will produce multiple independent outputs: `,`\n- Will ignores error if the type is unexpected: `?`\n- Array construction: `[]`\n- Object construction: `{}`\n- Concatenate or Add: `+`\n- Difference of sets or Substract: `-`\n- Size of selected element: `length`\n- Pipes are used to chain commands in a similar fashion than bash: `|`\n\n## Dealing with JSON objects\n\n- Display all keys: `keys`\n- Adds + 1 to all items: `map_values(.+1)`\n- Delete a key: `del(.foo)`\n- Convert an object to array: `to_entries | map([.key, .value])`\n\n## Dealing with fields\n\n- Concatenate two fields: `fieldNew=.field1+' '+.field2`\n\n## Dealing with arrays: Slicing and Filtering\n\n- All: `.[]`\n- First: `.[0]`\n- Range: `.[2:4]`\n- First 3: `.[:3]`\n- Last 2: `.[-2:]`\n- Before Last: `.[-2]`\n- Select array of int by value: `map(select(. >= 2))`\n- Select array of objects by value: `.[] | select(.id == \"second\")`\n- Select by type: `.[] | numbers`\n\nTypes can be `arrays`, `objects`, `iterables`, `booleans`, `numbers`, `normals`,\n `finites`, `strings`, `nulls`, `values` and `scalars`.\n\n## Dealing with arrays: Mapping and Transforming\n\n- Add + 1 to all items: `map(.+1)`\n- Delete 2 items: `del(.[1, 2])`\n- Concatenate arrays: `add`\n- Flatten an array: `flatten`\n- Create a range of numbers: `[range(2;4)]`\n- Display the type of each item: `map(type)`\n- Sort an array of basic type: `sort`\n- Sort an array of objects: `sort_by(.foo)`\n- Group by a key - opposite to flatten: `group_by(.foo)`\n- Minimum value of an array: `min` (see also `max`, `min_by(path_exp)`, `max_by(path_exp)`)\n- Remove duplicates: `unique` or `unique_by(.foo)` or `unique_by(length)`\n- Reverse an array: `reverse`\n\n", + "markdownDocumentation": "\nThis operator applies a jq filter to each value of a single input path and\nemits the filter's result as the value on the operator's output path.\n\nExactly one input is required - the operator fails if it is connected to no\ninput path or to more than one. Each value of that path is passed to the\nfilter independently, so a multi-valued path produces one filter run, and one\noutput value, per input value.\n\nA filter run can emit zero, one or several JSON values. If it emits exactly\none and **Output list with one item as string** is enabled (the default),\nthat single value is converted with Python's `str()`. For a plain JSON\nstring this yields the text unchanged, but for any other JSON type - numbers,\nbooleans, `null`, objects, arrays - it yields Python's own literal syntax\ninstead of JSON, for example `true` becomes `True` and `null` becomes `None`.\nThe same applies, regardless of the option, whenever a run emits zero or\nseveral values: the operator then outputs the Python `str()` of the whole\nresult list. Filters that are expected to always return a single JSON string\nare unaffected; filters that can return other types or a variable number of\nresults will need their output parsed back from this Python-literal form\ndownstream.\n\n> [jq](https://jqlang.org/) is like sed for JSON data - you can use it to\n> slice and filter and map and transform structured data with the same ease that sed, awk,\n> grep and friends let you play with text.\n\nIn order to test jq expressions, you can use [play.jqlang.org](https://play.jqlang.org/).\n\n## Basic concepts:\n\n- Filters separated by a comma will produce multiple independent outputs: `,`\n- Will ignores error if the type is unexpected: `?`\n- Array construction: `[]`\n- Object construction: `{}`\n- Concatenate or Add: `+`\n- Difference of sets or Substract: `-`\n- Size of selected element: `length`\n- Pipes are used to chain commands in a similar fashion than bash: `|`\n\n## Dealing with JSON objects\n\n- Display all keys: `keys`\n- Adds + 1 to all items: `map_values(.+1)`\n- Delete a key: `del(.foo)`\n- Convert an object to array: `to_entries | map([.key, .value])`\n\n## Dealing with fields\n\n- Concatenate two fields: `fieldNew=.field1+' '+.field2`\n\n## Dealing with arrays: Slicing and Filtering\n\n- All: `.[]`\n- First: `.[0]`\n- Range: `.[2:4]`\n- First 3: `.[:3]`\n- Last 2: `.[-2:]`\n- Before Last: `.[-2]`\n- Select array of int by value: `map(select(. >= 2))`\n- Select array of objects by value: `.[] | select(.id == \"second\")`\n- Select by type: `.[] | numbers`\n\nTypes can be `arrays`, `objects`, `iterables`, `booleans`, `numbers`, `normals`,\n `finites`, `strings`, `nulls`, `values` and `scalars`.\n\n## Dealing with arrays: Mapping and Transforming\n\n- Add + 1 to all items: `map(.+1)`\n- Delete 2 items: `del(.[1, 2])`\n- Concatenate arrays: `add`\n- Flatten an array: `flatten`\n- Create a range of numbers: `[range(2;4)]`\n- Display the type of each item: `map(type)`\n- Sort an array of basic type: `sort`\n- Sort an array of objects: `sort_by(.foo)`\n- Group by a key - opposite to flatten: `group_by(.foo)`\n- Minimum value of an array: `min` (see also `max`, `min_by(path_exp)`, `max_by(path_exp)`)\n- Remove duplicates: `unique` or `unique_by(.foo)` or `unique_by(length)`\n- Reverse an array: `reverse`\n\n", "pluginIcon": null, "properties": { "jq_expression": { @@ -19020,7 +20845,7 @@ "single_item_as_string": { "name": "single_item_as_string", "title": "Output list with one item as string", - "description": "", + "description": "If enabled, a filter result of exactly one value is written as that value's Python string representation instead of as a one-element list. If disabled, or if the filter returns zero or several values, the whole result list is written the same way.", "type": "string", "parameterType": "boolean", "value": "true", @@ -20238,6 +22063,10 @@ { "id": "validateRegex", "description": "The Validate regex plugin keeps the original value only when the full value matches the configured regular expression and otherwise fails validation. The Regex extract plugin returns match-derived output and can return an empty result when nothing matches." + }, + { + "id": "filterByRegex", + "description": "Filter by regex passes a value through exactly as it was, or excludes it \u2014 no partial output exists. Regex extract never returns the value itself; its output is always a fragment of the match, a substring or a capturing group." } ] }, @@ -20303,6 +22132,10 @@ { "id": "replace", "description": "The Replace plugin swaps one fixed substring for another everywhere it occurs. The Regex replace plugin does the same global substitution, but what counts as a hit is described by a regular expression." + }, + { + "id": "filterByRegex", + "description": "Filter by regex decides which values make it into the output, but never alters what a kept value contains. Regex replace keeps every value without exception; what changes is the content itself." } ] }, @@ -20354,7 +22187,7 @@ }, { "id": "filterByRegex", - "description": "The Regex selection plugin keeps the checked value out of the output and instead returns a pattern-list-shaped result filled with the provided output value where a pattern matches. The Filter by regex plugin keeps or drops values from the input sequence based on full-string matching." + "description": "The Regex selection plugin keeps the checked value out of the output and instead returns a pattern-list-shaped result filled with the provided output value where a pattern matches. The Filter by regex plugin keeps or drops values from the input sequence based on regex matching." } ] }, @@ -20671,6 +22504,10 @@ { "id": "removeDuplicates", "description": "The two plugins filter on different grounds. Remove values drops a value because of what it is; Remove duplicates drops a value because it already appeared earlier in the sequence." + }, + { + "id": "filterByRegex", + "description": "Every value Remove values can exclude has to appear in the blacklist by name. Filter by regex can exclude many values through one pattern, without naming any of them." } ] }, @@ -22174,6 +24011,10 @@ { "id": "regexExtract", "description": "Regex extract turns the match into output by returning the matched substring or the first capturing group. Validate regex leaves the value unchanged and only lets it through when the full value matches the pattern." + }, + { + "id": "filterByRegex", + "description": "A non-matching value is excluded by Filter by regex; every other value continues. One mismatch fails the whole check under Validate regex." } ] }, @@ -22260,6 +24101,10 @@ { "id": "concatPairwise", "description": "Zip handles unequal input lengths by padding, not truncating, and is constrained to exactly two inputs. Concatenate pairwise removes that constraint \u2014 it accepts any number of inputs \u2014 but resolves the length mismatch by stopping at the shortest." + }, + { + "id": "inputHash", + "description": "Zip pairs exactly two inputs position by position and returns one output per pair, using the same optional separator between values. Combined input hash instead combines every connected value into a single hash, regardless of how many ports are used." } ] }, diff --git a/docs/automate/cmemc-command-line-interface/backup-and-restore/index.md b/docs/automate/cmemc-command-line-interface/backup-and-restore/index.md index 93ce8b1f2..938f3f394 100644 --- a/docs/automate/cmemc-command-line-interface/backup-and-restore/index.md +++ b/docs/automate/cmemc-command-line-interface/backup-and-restore/index.md @@ -9,7 +9,7 @@ tags: ## Introduction -cmemc can export and re-import all data which Corporate Memory manages: the knowledge graphs in the store, the projects in the workspace, and any subset of them. +cmemc can export and re-import all data which eccenca Corporate Memory manages: the knowledge graphs in the store, the projects in the workspace, and any subset of them. This page describes which artifacts you can create, how to restore each of them, and - just as important - which parts of a deployment cmemc does **not** cover. !!! warning "A cmemc export is not a complete instance backup" diff --git a/docs/automate/cmemc-command-line-interface/command-reference/admin/logs/index.md b/docs/automate/cmemc-command-line-interface/command-reference/admin/logs/index.md new file mode 100644 index 000000000..309ac1465 --- /dev/null +++ b/docs/automate/cmemc-command-line-interface/command-reference/admin/logs/index.md @@ -0,0 +1,113 @@ +--- +title: "cmemc: Command Group - admin logs" +description: "List log lines and get the state of the log buffers." +icon: material/text-box-search-outline +tags: + - cmemc + - Logging + - Debugging +--- + +# admin logs Command Group + + + +List log lines and get the state of the log buffers. + +Each backend of Corporate Memory which keeps its most recent log lines in an in-memory buffer is a job here, under the same job names the `admin metrics` commands use. This command group reads those buffers, which is the fastest way to find out what a failing request did on the server. + +!!! note + SPARQL queries have their own, more specific command: `query status` gives you information about the last executed queries. + + + +## admin logs list + +List the most recent log lines of the jobs. + +```shell-session title="Usage" +cmemc admin logs list [OPTIONS] +``` + + + + +Outputs the log lines which the jobs still hold in their in-memory buffer, newest last. Use the ``--filter`` option to narrow the output down; the `level`, `logger` and `contains` filters are applied by the job, so the ``--limit`` counts the lines which match them. A filter which can be given more than once requires all of its values to match. + +The TIME column of the table shows your local time and names its offset, while the ``--plain`` and ``--raw`` output use the UTC timestamps the jobs sent. A `(+)` in front of a message marks a line which carries a stack trace; use ``--plain`` or ``--raw`` to read it. + +!!! note + Each instance of a job buffers its own lines, so in a deployment with more than one instance the answer comes from whichever instance was asked, which the output names. Use the `admin logs status` command to see how far back a buffer reaches. + + +!!! note + With `--follow` on a terminal, the live table is a viewport rather than a result set: it is filled to the height of your terminal and therefore shows as many lines as fit, which can be more than `--limit`. + + +!!! note + To investigate SPARQL queries, the `query status` command is the better tool: it gives you information about the last executed queries, instead of whatever the jobs happened to log about them. + + + + +??? info "Options" + ```text + + --filter ... Filter log lines by one of the following + filter names and a corresponding value: + level, logger, contains, job, newer-than, + older-than. + --limit INTEGER RANGE How many lines to output. The lines of all + jobs are merged and the newest ones are + shown, so this counts across jobs. Use 0 to + get as many lines as the jobs are willing to + return. [default: 20; x>=0] + --plain Outputs plain log lines instead of a table. + Nothing is shortened, the timestamps are the + UTC ones the jobs sent, and a stack trace is + output as an indented block below its line. + --raw Outputs raw JSON. + --follow Keeps the command running and outputs new + log lines as they arrive. On a terminal, the + table is updated in place and filled to the + height of the terminal, so it can show more + lines than --limit; otherwise, lines are + appended. Can not be combined with the + older-than filter, which looks at the past. + --polling-interval INTEGER RANGE + How many seconds to wait between two log + reads while following. Applies together with + --follow only, and is ignored when a read + reports that more lines are already waiting. + [default: 1; 0<=x<=60] + ``` + +## admin logs status + +Get the state of the in-memory log buffers. + +```shell-session title="Usage" +cmemc admin logs status [OPTIONS] +``` + + + + +Outputs whether log lines are captured at all, at which level, how many are held, which time span they cover and which loggers are excluded. Use this command to find out why the `admin logs list` command returns fewer lines than expected, or none. + +The buffers of all jobs are shown as one structure, with the job as the first level of every key. Use ``--key` build.capacity` to address a single value of a single job, and ``--key`build` to see one job only. + +!!! note + `firstTimestamp` and `lastTimestamp` are not part of the answer of a job. The status of a buffer names the sequence numbers it spans but no times, so cmemc reads the oldest and the newest buffered line to tell you how far back the buffer reaches. They are shown in your local time, as the TIME column of the `admin logs list` table is, so that the two can be compared; `--raw` keeps the UTC the jobs sent. + + + + +??? info "Options" + ```text + + --raw Outputs raw JSON. + --key TEXT Get a specific key only. If the given value is the prefix of + more than one key, the table is reduced to these keys. Use `all` + to output the complete table. + ``` diff --git a/docs/automate/cmemc-command-line-interface/command-reference/dataset/index.md b/docs/automate/cmemc-command-line-interface/command-reference/dataset/index.md index cdf8f88e9..169a9e746 100644 --- a/docs/automate/cmemc-command-line-interface/command-reference/dataset/index.md +++ b/docs/automate/cmemc-command-line-interface/command-reference/dataset/index.md @@ -87,7 +87,7 @@ This command deletes existing datasets in integration projects from Corporate Me Download the resource file of a dataset. ```shell-session title="Usage" -$ cmemc dataset download [OPTIONS] DATASET_ID OUTPUT_PATH +$ cmemc dataset download [OPTIONS] DATASET_ID [OUTPUT_PATH] ``` @@ -95,7 +95,7 @@ $ cmemc dataset download [OPTIONS] DATASET_ID OUTPUT_PATH This command downloads the file resource of a dataset to your local file system or to standard out (`-`). Note that this is not possible for dataset types such as Knowledge Graph (`eccencaDataplatform`) or SQL endpoint (`sqlEndpoint`). -Without providing an output path, the output file name will be the same as the remote file resource. +Without providing an output path, the output file name will be the same as the remote file resource (without its directories) and the file is saved to the current directory. Use the ``--filename-template`` option to build the output file name from placeholders instead. !!! note Datasets can be listed by using the `dataset list` command. @@ -106,8 +106,16 @@ Without providing an output path, the output file name will be the same as the r ??? info "Options" ```text - --replace Replace existing files. This is a dangerous option, so use it - with care! + -t, --filename-template TEXT Template for the output file name, used + instead of an output path. Possible + placeholders are (Jinja2): {{file}} (the name + of the remote file resource), {{project}} (the + project ID), {{dataset}} (the dataset ID), + {{connection}} (from the --connection option) + and {{date}} (the current date as YYYY-MM-DD). + Needed directories will be created. + --replace Replace existing files. This is a dangerous + option, so use it with care! ``` ## dataset upload diff --git a/docs/automate/cmemc-command-line-interface/command-reference/index.md b/docs/automate/cmemc-command-line-interface/command-reference/index.md index 61f09160a..b8f2f0a47 100644 --- a/docs/automate/cmemc-command-line-interface/command-reference/index.md +++ b/docs/automate/cmemc-command-line-interface/command-reference/index.md @@ -33,6 +33,8 @@ tags: | [admin client](admin/client/index.md) | [list](admin/client/index.md#admin-client-list) | List client accounts. | | [admin client](admin/client/index.md) | [secret](admin/client/index.md#admin-client-secret) | Get or generate a new secret for a client account. | | [admin client](admin/client/index.md) | [open](admin/client/index.md#admin-client-open) | Open clients in the browser. | +| [admin logs](admin/logs/index.md) | [list](admin/logs/index.md#admin-logs-list) | List the most recent log lines of the jobs. | +| [admin logs](admin/logs/index.md) | [status](admin/logs/index.md#admin-logs-status) | Get the state of the in-memory log buffers. | | [admin metrics](admin/metrics/index.md) | [get](admin/metrics/index.md#admin-metrics-get) | Get sample data of a metric. | | [admin metrics](admin/metrics/index.md) | [inspect](admin/metrics/index.md#admin-metrics-inspect) | Inspect a metric. | | [admin metrics](admin/metrics/index.md) | [list](admin/metrics/index.md#admin-metrics-list) | List metrics for a specific job. | @@ -100,6 +102,7 @@ tags: | [graph validation](graph/validation/index.md) | [export](graph/validation/index.md#graph-validation-export) | Export a report of finished validations. | | [package](package/index.md) | [inspect](package/index.md#package-inspect) | Inspect the manifest of a package. | | [package](package/index.md) | [list](package/index.md#package-list) | List installed packages. | +| [package](package/index.md) | [dependencies](package/index.md#package-dependencies) | Show the dependency tree of the installed packages. | | [package](package/index.md) | [install](package/index.md#package-install) | Install packages. | | [package](package/index.md) | [download](package/index.md#package-download) | Download a package archive from the marketplace. | | [package](package/index.md) | [uninstall](package/index.md#package-uninstall) | Uninstall installed packages. | @@ -121,11 +124,21 @@ tags: | [project file](project/file/index.md) | [upload](project/file/index.md#project-file-upload) | Upload a file to a project. | | [project file](project/file/index.md) | [inspect](project/file/index.md#project-file-inspect) | Display all metadata of a file resource. | | [project file](project/file/index.md) | [usage](project/file/index.md#project-file-usage) | Display all usage data of a file resource. | -| [project variable](project/variable/index.md) | [list](project/variable/index.md#project-variable-list) | List available project variables. | -| [project variable](project/variable/index.md) | [get](project/variable/index.md#project-variable-get) | Get the value or other data of a project variable. | -| [project variable](project/variable/index.md) | [delete](project/variable/index.md#project-variable-delete) | Delete project variables. | -| [project variable](project/variable/index.md) | [create](project/variable/index.md#project-variable-create) | Create a new project variable. | -| [project variable](project/variable/index.md) | [update](project/variable/index.md#project-variable-update) | Update data of an existing project variable. | +| [project rule-block](project/rule-block/index.md) | [list](project/rule-block/index.md#project-rule-block-list) | List available rule blocks. | +| [project rule-block](project/rule-block/index.md) | [inspect](project/rule-block/index.md#project-rule-block-inspect) | Display all metadata of a rule block. | +| [project rule-block](project/rule-block/index.md) | [open](project/rule-block/index.md#project-rule-block-open) | Open rule blocks in the browser. | +| [project rule-block](project/rule-block/index.md) | [usage](project/rule-block/index.md#project-rule-block-usage) | Display all tasks which use a rule block. | +| [project rule-block](project/rule-block/index.md) | [create](project/rule-block/index.md#project-rule-block-create) | Create an empty rule block. | +| [project rule-block](project/rule-block/index.md) | [update](project/rule-block/index.md#project-rule-block-update) | Update a key of an existing rule block. | +| [project rule-block](project/rule-block/index.md) | [export](project/rule-block/index.md#project-rule-block-export) | Export rule blocks to JSON files. | +| [project rule-block](project/rule-block/index.md) | [import](project/rule-block/index.md#project-rule-block-import) | Import rule blocks from JSON files. | +| [project rule-block](project/rule-block/index.md) | [delete](project/rule-block/index.md#project-rule-block-delete) | Delete rule blocks. | +| [project rule-block](project/rule-block/index.md) | [eval](project/rule-block/index.md#project-rule-block-eval) | Evaluate a rule block against its stored input examples. | +| [project variable](project/variable/index.md) | [list](project/variable/index.md#project-variable-list) | List available project, execution and global variables. | +| [project variable](project/variable/index.md) | [get](project/variable/index.md#project-variable-get) | Get the value or other data of a project, execution or global variable. | +| [project variable](project/variable/index.md) | [delete](project/variable/index.md#project-variable-delete) | Delete project and execution variables. | +| [project variable](project/variable/index.md) | [create](project/variable/index.md#project-variable-create) | Create a new project or execution variable. | +| [project variable](project/variable/index.md) | [update](project/variable/index.md#project-variable-update) | Update data of an existing project or execution variable. | | [query](query/index.md) | [execute](query/index.md#query-execute) | Execute queries which are loaded from files or a query catalog. | | [query](query/index.md) | [list](query/index.md#query-list) | List available queries from a query catalog. | | [query](query/index.md) | [open](query/index.md#query-open) | Open queries in the editor of a query catalog in your browser. | @@ -147,6 +160,7 @@ tags: | [workflow](workflow/index.md) | [io](workflow/index.md#workflow-io) | Execute a workflow with file input/output. | | [workflow](workflow/index.md) | [list](workflow/index.md#workflow-list) | List available workflow. | | [workflow](workflow/index.md) | [status](workflow/index.md#workflow-status) | Get status information of workflow(s). | +| [workflow](workflow/index.md) | [variables](workflow/index.md#workflow-variables) | List the execution variables a workflow run needs. | | [workflow](workflow/index.md) | [open](workflow/index.md#workflow-open) | Open a workflow in your browser. | | [workflow scheduler](workflow/scheduler/index.md) | [open](workflow/scheduler/index.md#workflow-scheduler-open) | Open scheduler(s) in the browser. | | [workflow scheduler](workflow/scheduler/index.md) | [list](workflow/scheduler/index.md#workflow-scheduler-list) | List available scheduler. | diff --git a/docs/automate/cmemc-command-line-interface/command-reference/package/index.md b/docs/automate/cmemc-command-line-interface/command-reference/package/index.md index c20de8c39..1a5e9cccb 100644 --- a/docs/automate/cmemc-command-line-interface/command-reference/package/index.md +++ b/docs/automate/cmemc-command-line-interface/command-reference/package/index.md @@ -16,10 +16,10 @@ List, (un)install, download, export, create, or inspect packages. Packages bundle re-usable content such as projects or vocabularies together with a manifest, so that it can be distributed and installed in other deployments. Packages are identified by a `PACKAGE_ID` and are used either as a package directory, as a Corporate Memory Package Archive (.cpa), or from a marketplace. -Typical workflows are to install packages from a marketplace (`package search` and `package install`), to fetch a package archive without installing it (`package download`), and to create packages from your own workspace (`package export`, `package build` and `package publish`). +Typical workflows are to install packages from a marketplace (`package search` and `package install`), to check what a package pulls in and whether the installed packages fit together (`package dependencies`), to fetch a package archive without installing it (`package download`), and to create packages from your own workspace (`package export`, `package build` and `package publish`). !!! note - To get a list of installed packages, execute the `package list` command or use tab-completion. + To get a list of installed packages, execute the `package list` command or use tab-completion. `package dependencies` shows the dependency tree of the installed packages and where it is broken. @@ -79,6 +79,33 @@ Outputs a table of the packages which are currently installed in your Corporate --raw Outputs raw JSON. ``` +## package dependencies + +Show the dependency tree of the installed packages. + +```shell-session title="Usage" +cmemc package dependencies [OPTIONS] [PACKAGE_ID] +``` + + + + +One tree is shown per installed package which no other installed package requires, expanded into the packages it pulls in - marketplace packages and Python packages alike - each with its type (`project`, `vocabulary` or `python`), the declared version constraint, the version installed in your Corporate Memory and whether the dependency is `satisfied`, `violated` or `missing`. A dependency is `violated` when the installed version lies outside of the constraint, and `missing` when it is not installed at all, for example because it was uninstalled although the package needs it, or because the package was installed while ignoring its dependencies. A package which appears a second time is shown as `[see above]`, a dependency cycle ends in `[cycle]`. With a `PACKAGE_ID` only the tree of that installed package is shown. A consistent tree ends with no warning; reinstalling a dependency at a satisfying version repairs a broken one. + +The raw JSON output carries both forms of the tree: `packages` lists every package once with its dependencies and the packages which depend on it (`required_by`), and `tree` is the nested form; every violated and missing dependency is collected under `violations` - an empty list means the tree is consistent. + +!!! note + `package install` refuses an installation which would break the installed tree unless `--force-dependencies` is given. + + + + +??? info "Options" + ```text + + --raw Outputs raw JSON. + ``` + ## package install Install packages. @@ -94,6 +121,12 @@ This command installs a package either from the marketplace or from local packag If a local package is chosen which has unzipped project directories, the installation will handle the zipping silently. See the `package export` command for more information. +Dependencies are installed at the newest version which satisfies the version constraints declared on them. If an installed dependency violates such a constraint, the installation fails before anything is changed - use ``--force-dependencies`` to upgrade or downgrade it instead. Use this option with care: it breaks every installed package which needs the same dependency at a different version, and the command warns about each of them. + +Use ``--skip-installed`` to make repeated installations cheap: a package which is already installed in the requested version is left untouched, including its dependencies. If another version is installed, the installation is not skipped - combine the option with ``--replace`` to upgrade or downgrade in this case. + +An installation (also a replacement) whose versions would violate the version constraint of an already installed package is refused before anything is changed, naming each affected package, its constraint and the version which would break it. ``--force-dependencies`` installs anyway and warns about every installed package it broke; the `package dependencies` command shows where the installed tree is broken. Replacing a package reinstalls a dependency only it held at a version satisfying its constraint, never at the latest published version. + ??? info "Options" @@ -103,6 +136,9 @@ If a local package is chosen which has unzipped project directories, the install directory. --replace Replace (overwrite) an existing package version or package content, if present. + --skip-installed Skip the installation if the package is already + installed in the requested version. For local + packages, the version of the manifest is compared. --no-cache Disable using cached package versions. --ignore-lock Ignore and release the package lock file for this operation. Use this to recover from a stale lock @@ -111,6 +147,16 @@ If a local package is chosen which has unzipped project directories, the install processes rely on); use with care. --version TEXT Specific version to install from the marketplace. Defaults to the latest version. + --force-dependencies Upgrade or downgrade an installed dependency whose + version violates the constraint the package declares + on it, instead of failing the installation, and + install the package although a version it puts in + place - its own, or a dependency's - violates the + constraint of an installed package. Such an + installation is refused without this option; with + it, a warning names every installed package it + broke. This is a dangerous option, so use it with + care. --marketplace-url TEXT Base URL of the Marketplace - uses environment variable ECCENCA_MARKETPLACE_URL if available. [default: https://eccenca.market] @@ -242,6 +288,10 @@ This command processes a package directory, validates its content including the If the package contains an extracted project (directory) instead of a ZIP, it is zipped automatically in a temporary copy — the original package directory is never modified. The manifest still need to reference the project ZIP. See the `package export` command for more information. +The archive manifest carries a `build_timestamp`: the time of the build in UTC, as an ISO 8601 date-time with second precision. It is set only when the manifest in the package directory does not carry one; a timestamp written by the package author is kept unchanged. The manifest file in the package directory is never modified by the build. + +An application view declared in the manifest (file type `view`) is checked against its file: the JSON file written by `admin view export` has to hold exactly one view, whose id is the declared `view_id`. The `default` view of a deployment cannot be shipped. + !!! note Package archives can be published to the marketplace using the `package publish` command. diff --git a/docs/automate/cmemc-command-line-interface/command-reference/project/rule-block/index.md b/docs/automate/cmemc-command-line-interface/command-reference/project/rule-block/index.md new file mode 100644 index 000000000..b8eaca899 --- /dev/null +++ b/docs/automate/cmemc-command-line-interface/command-reference/project/rule-block/index.md @@ -0,0 +1,366 @@ +--- +title: "cmemc: Command Group - project rule-block" +description: "List, inspect, create, delete, import or export project rule blocks." +icon: material/function-variant +tags: + - Transform + - cmemc +--- + +# project rule-block Command Group + + + +List, inspect, create, delete, import or export project rule blocks. + +A rule block is a named, reusable operator tree with declared input ports, comparable to a function in a programming language. Transform and linking rules reference a rule block instead of repeating its operators, so a correction made in the rule block reaches every rule that uses it. + +Rule blocks are identified by a `RULE_BLOCK_ID`, which is a concatenation of a `PROJECT_ID` and the ID of the rule block, such as `my-project:normalize-name`. + +!!! note + To get a list of existing rule blocks, execute the `list` command or use tab-completion. Rule blocks need Corporate Memory 26.2 or newer. + + + +## project rule-block list + +List available rule blocks. + +```shell-session title="Usage" +cmemc project rule-block list [OPTIONS] +``` + + + + +Output and filter a list of available rule blocks. Each rule block is listed with its ID, its number of input ports and its label. + +Filters are applied in the order they are given on the command line. The `usage` filter resolves the dependents of each remaining rule block, so putting a cheaper filter such as `project` in front of it reduces the number of requests it has to make. + + + +??? info "Options" + ```text + + --filter ... Filter rule blocks by one of the following filter + names and a corresponding value: project, regex, + tag, usage, deprecated. + --raw Outputs raw JSON objects of the rule blocks. + --id-only Lists only rule block IDs and no labels or other + metadata. This is useful for piping the IDs into + other cmemc commands. + ``` + +## project rule-block inspect + +Display all metadata of a rule block. + +```shell-session title="Usage" +cmemc project rule-block inspect [OPTIONS] RULE_BLOCK_ID +``` + + + + +The complete rule block is shown as a flat key and value table: its metadata, each declared input port with its label, display order and deprecation state, and the operator tree. + +For accessing nested values, use the following notation: `ruleBlockModel.ports[0].label` + +!!! note + Some shell environments require quotes around expressions with square brackets. + + +```shell-session title="Example" +cmemc project rule-block inspect my-project:normalize-name +``` + + +```shell-session title="Example" +cmemc project rule-block inspect my-project:normalize-name --key "ruleBlockModel.ports[0].label" +``` + + + + + +??? info "Options" + ```text + + --key TEXT Get a specific key only. If the given value is the prefix of + more than one key, the table is reduced to these keys. Use `all` + to output the complete table. + --raw Outputs raw JSON. + ``` + +## project rule-block open + +Open rule blocks in the browser. + +```shell-session title="Usage" +cmemc project rule-block open RULE_BLOCK_IDS... +``` + + + + +With this command, you can open a rule block in the Rule Block Editor in your browser. + +The command accepts multiple rule block IDs which results in opening multiple browser tabs. + + + +## project rule-block usage + +Display all tasks which use a rule block. + +```shell-session title="Usage" +cmemc project rule-block usage [OPTIONS] RULE_BLOCK_ID +``` + + + + +Tasks which reference the rule block directly, such as a transformation with a rule block operator, are listed as direct users. Tasks which reference one of those in turn, such as a workflow containing that transformation, are listed as indirect users. Deleting the rule block together with its dependent tasks deletes all of them. + + + +??? info "Options" + ```text + + --raw Outputs raw JSON. + ``` + +## project rule-block create + +Create an empty rule block. + +```shell-session title="Usage" +cmemc project rule-block create [OPTIONS] RULE_BLOCK_ID +``` + + + + +The new rule block has no input ports and no operator tree yet. Open it with the `open` command to build it in the Rule Block Editor, or fill it with the `import` command from an exported file. + +```shell-session title="Example" +cmemc project rule-block create normalize-name --project my-project --label "Normalize Name" +``` + + + + +??? info "Options" + ```text + + --project TEXT The project, where the rule block will be created. If + there is only one project in the workspace, this option + can be omitted. + --label TEXT The label of the new rule block. + --description TEXT The description of the new rule block. + ``` + +## project rule-block update + +Update a key of an existing rule block. + +```shell-session title="Usage" +cmemc project rule-block update [OPTIONS] RULE_BLOCK_ID +``` + + + + +Any key shown by the `inspect` command can be updated, including nested keys such as the label or the deprecation state of an input port. All other values are preserved. + +!!! note + A new input port cannot be added this way, because a list index that does not exist yet cannot be addressed. Add input ports in the Rule Block Editor, or use `export`, edit the file, and `import --replace`. + + +```shell-session title="Example" +cmemc project rule-block update my-project:normalize-name --key label --value "Normalize Person Name" +``` + + +```shell-session title="Example" +cmemc project rule-block update my-project:normalize-name --key "ruleBlockModel.ports[1].deprecated" --value true +``` + + + + +??? info "Options" + ```text + + --key TEXT The key to update. Supports nested keys using 'a.b[i].c' + notation, e.g. ruleBlockModel.ports[0].deprecated. [required] + --value TEXT The new value. Parsed as JSON when possible (e.g. true, false, + 1), otherwise used as a plain string. [required] + ``` + +## project rule-block export + +Export rule blocks to JSON files. + +```shell-session title="Usage" +$ cmemc project rule-block export [OPTIONS] [RULE_BLOCK_IDS]... +``` + + + + +The exported file holds the rule block as a task structure without its project, so it can be imported into any project with the `import` command. + +!!! note + The exported rule block is not necessarily self-contained. Its operator parameters can reference project variables and custom transformer plugins, and neither of those travels with the file, so an import into a different project can succeed and still leave a rule block which does not work. + + +```shell-session title="Example" +$ cmemc project rule-block export my-project:normalize-name --output-file normalize-name.json +``` + + +```shell-session title="Example" +$ cmemc project rule-block export --filter project my-project --output-dir backup -t "{{project}}/{{id}}" +``` + + + + +??? info "Options" + ```text + + -a, --all Export all rule blocks. + --filter ... Filter rule blocks by one of the following + filter names and a corresponding value: + project, regex, tag, usage, deprecated. + --output-file FILE Export to this file. Use '-' for stdout. Only + allowed with a single rule block. If + specified, overrides --output-dir and + --filename-template. + --output-dir DIRECTORY The base directory where the export files will + be created. Ignored if --output-file is + specified. [default: .] + -t, --filename-template TEXT Template for the export file name(s). Possible + placeholders are (Jinja2): {{id}} (the rule + block ID), {{project}} (the project ID), + {{connection}} (from the --connection option) + and {{date}} (the current date as YYYY-MM-DD). + The file suffix will be appended. Needed + directories will be created. [default: {{date + }}-{{connection}}-{{project}}-{{id}}.rule- + block] + --replace Replace existing export files. This is a + dangerous option, so use it with care. + --without-userdata Exclude user-identifying metadata (creation + and modification timestamps and account names) + from the exported files. + ``` + +## project rule-block import + +Import rule blocks from JSON files. + +```shell-session title="Usage" +cmemc project rule-block import [OPTIONS] INPUT_FILES... +``` + + + + +This command imports rule blocks from JSON files created with the `export` command. A file does not record its source project, so the same file can be imported into any project. + +!!! note + The imported rule block is not necessarily self-contained. Its operator parameters can reference project variables and custom transformer plugins, and neither of those travels with the file. + + +```shell-session title="Example" +cmemc project rule-block import normalize-name.json --project my-other-project +``` + + + + +??? info "Options" + ```text + + --project TEXT The project, where the rule blocks will be imported. If + there is only one project in the workspace, this option can + be omitted. + --replace Replace existing rule blocks. By default, import will skip + rule blocks that already exist. + --id TEXT Import the rule block under this ID instead of the one + stored in the file. + ``` + +## project rule-block delete + +Delete rule blocks. + +```shell-session title="Usage" +cmemc project rule-block delete [OPTIONS] [RULE_BLOCK_IDS]... +``` + + + + +Deleting a rule block which other tasks reference is refused, because DataIntegration would leave those tasks pointing at something that no longer exists and would report the breakage nowhere. Use the ``--with-dependent-tasks`` option to delete the rule block together with every task which references it, directly or indirectly. Use the `usage` command to see who those are beforehand. + +When several rule blocks are selected, all of them are checked first, and nothing is deleted if one of them is used. + +```shell-session title="Example" +cmemc project rule-block delete my-project:normalize-name +``` + + +```shell-session title="Example" +cmemc project rule-block delete --filter usage unused +``` + + + + +??? info "Options" + ```text + + -a, --all Delete all rule blocks. This is a dangerous option, + so use it with care. + --filter ... Filter rule blocks by one of the following filter + names and a corresponding value: project, regex, + tag, usage, deprecated. + --with-dependent-tasks Also delete every task which directly or indirectly + references the rule block. This is a dangerous + option, so use it with care. + ``` + +## project rule-block eval + +Evaluate a rule block against its stored input examples. + +```shell-session title="Usage" +cmemc project rule-block eval [OPTIONS] RULE_BLOCK_ID +``` + + + + +The input examples are the ones stored with the rule block in the Rule Block Editor. Each example is listed with the values it feeds into the input ports and the values the rule block produces from them. An example which fails to evaluate shows the failing operator and its error message instead of a result. + +Use ``--raw`` to get the complete evaluation tree, which also carries the intermediate values of every operator and the stack trace of a failure. + +!!! note + A rule block can only be evaluated as part of a project. There is no way to evaluate an exported file without importing it first. + + +```shell-session title="Example" +cmemc project rule-block eval my-project:normalize-name +``` + + + + +??? info "Options" + ```text + + --raw Outputs raw JSON. + ``` diff --git a/docs/automate/cmemc-command-line-interface/command-reference/project/variable/index.md b/docs/automate/cmemc-command-line-interface/command-reference/project/variable/index.md index 706adf634..dfece9181 100644 --- a/docs/automate/cmemc-command-line-interface/command-reference/project/variable/index.md +++ b/docs/automate/cmemc-command-line-interface/command-reference/project/variable/index.md @@ -1,6 +1,6 @@ --- title: "cmemc: Command Group - project variable" -description: "List, create, delete or get data from project variables." +description: "List, create, delete or get data from project and execution variables." icon: material/variable-box tags: - Variables @@ -11,16 +11,24 @@ tags: -List, create, delete or get data from project variables. +List, create, delete or get data from project and execution variables. -Project variables can be used in dataset and task parameters, and in the template transform operator. Variables are either based on a static value or based on a template. They may use templates that access globally configured variables or other preceding variables from the same project. +Project variables can be used in dataset and task parameters, and in the template transform operator. Execution variables are declared on a single task of a project, such as a workflow or a dataset. Global variables are defined in the configuration of Corporate Memory. They are listed and can be read, but not changed. Variables are either based on a static value or based on a template. They may use templates that access globally configured variables or other preceding variables from the same project. + +Variables are identified by a `VARIABLE_ID`. To get a list of existing variables, execute the list command or use tab-completion. The `VARIABLE_ID` of a project variable is a concatenation of a `PROJECT_ID` and a `VARIABLE_NAME`, such as `my-project:my-variable`. The `VARIABLE_ID` of an execution variable adds the `TASK_ID`, such as `my-project:my-task:my-variable`. The `VARIABLE_ID` of a global variable is its `VARIABLE_NAME`. + +!!! note + Variables can be flagged as sensitive. Corporate Memory does not hand out the value and the template of such a variable, they are only available to other sensitive variables of the same scope and to task parameters which accept sensitive variables, such as password parameters. + + +!!! note + The default value of an execution variable can be overridden for a single run with the `--parameter` option of the `workflow execute` and `workflow io` commands. Workflows started by a scheduler can not take such overrides, they always use the default values. -Variables are identified by a `VARIABLE_ID`. To get a list of existing variables, execute the list command or use tab-completion. The `VARIABLE_ID` is a concatenation of a `PROJECT_ID` and a `VARIABLE_NAME`, such as `my-project:my-variable`. ## project variable list -List available project variables. +List available project, execution and global variables. ```shell-session title="Usage" cmemc project variable list [OPTIONS] @@ -29,7 +37,11 @@ cmemc project variable list [OPTIONS] -Outputs a table or a list of project variables. +Outputs a table or a list of project variables, the execution variables of tasks and the global variables. The table states the scope of each variable (project, execution or global) and, for execution variables, the label of the task they are declared on. + +Sensitive variables are marked and listed without their value and template, since Corporate Memory does not hand them out. + +Errors of single variables, such as a template which can not be resolved, are shown as warnings below the table. @@ -40,14 +52,14 @@ Outputs a table or a list of project variables. --id-only Lists only variables names and no other metadata. This is useful for piping the IDs into other commands. - --filter ... Filter project variables by one of the following - filter names and a corresponding value: project, - regex. + --filter ... Filter variables by one of the following filter + names and a corresponding value: project, scope, + sensitive, type, task, regex. ``` ## project variable get -Get the value or other data of a project variable. +Get the value or other data of a project, execution or global variable. ```shell-session title="Usage" cmemc project variable get [OPTIONS] VARIABLE_ID @@ -62,6 +74,8 @@ Use the ``--key`` option to specify which information you want to get. Only the `value` key is always available on a project variable. Static value variables have no `template` key, and the `description` key is optional for both types of variables. Use `--raw` to access all fields. +The value and the template of a sensitive variable can not be read, only its description is available. + ??? info "Options" @@ -75,7 +89,7 @@ Use the ``--key`` option to specify which information you want to get. ## project variable delete -Delete project variables. +Delete project and execution variables. ```shell-session title="Usage" cmemc project variable delete [OPTIONS] [VARIABLE_IDS]... @@ -88,6 +102,10 @@ There are three selection mechanisms: with specific IDs - only those specified v Variables are automatically sorted by their dependencies and deleted in the correct order (template-based variables that depend on others are deleted first, then their dependencies). +Global variables can not be deleted. If `--all` or `--filter` select them, they are skipped. + +An execution variable of a workflow is deleted even if tasks of the workflow still use it. In this case, a warning names these tasks, since a run of the workflow then needs the variable as parameter. + ??? info "Options" @@ -95,14 +113,14 @@ Variables are automatically sorted by their dependencies and deleted in the corr -a, --all Delete all variables. This is a dangerous option, so use it with care. - --filter ... Filter project variables by one of the following - filter names and a corresponding value: project, - regex. + --filter ... Filter variables by one of the following filter + names and a corresponding value: project, scope, + sensitive, type, task, regex. ``` ## project variable create -Create a new project variable. +Create a new project or execution variable. ```shell-session title="Usage" cmemc project variable create [OPTIONS] VARIABLE_NAME @@ -111,13 +129,18 @@ cmemc project variable create [OPTIONS] VARIABLE_NAME -Variables need to be created with a value or a template (not both). In addition to that, a project ID and a name are mandatory. +Variables need to be created with a value or a template (not both). In addition to that, a project ID and a name are mandatory. Use the ``--task`` option to create an execution variable on a task instead of a project variable. The task is given as `PROJECT_ID`:`TASK_ID`, so the ``--project`` option is not used then. ```shell-session title="Example" cmemc project variable create my_var --project my_project --value abc ``` +```shell-session title="Example" +cmemc project variable create my_var --task my_project:my_workflow --value abc +``` + + !!! note cmemc is currently not able to manage the order of the variables in a project. This means you have to create plain value variables in advance, before you can create template based variables, which access these values. @@ -127,20 +150,27 @@ cmemc project variable create my_var --project my_project --value abc ??? info "Options" ```text - --value TEXT The value of the new project variable. - --template TEXT The template of the new project variable. You can use - Jinja template syntax, e.g. use '{{global.myVar}}' for + --value TEXT The value of the new variable. + --template TEXT The template of the new variable. You can use Jinja + template syntax, e.g. use '{{global.myVar}}' for accessing global variables, or '{{project.myVar}}' for accessing variables from the same project. - --description TEXT The optional description of the new project variable. - --project TEXT The project, where you want to create the variable in. - If there is only one project in the workspace, this - option can be omitted. + --description TEXT The optional description of the new variable. + --sensitive Flag the new variable as sensitive. The value and the + template of a sensitive variable can not be read back, + they are only available to other sensitive variables and + to task parameters which accept sensitive variables. + --project TEXT The project, where you want to create the project + variable in. If there is only one project in the + workspace, this option can be omitted. + --task TEXT The task, where you want to create an execution variable + on, given as PROJECT_ID:TASK_ID. Without this option, a + project variable is created. ``` ## project variable update -Update data of an existing project variable. +Update data of an existing project or execution variable. ```shell-session title="Usage" cmemc project variable update [OPTIONS] VARIABLE_ID @@ -149,21 +179,30 @@ cmemc project variable update [OPTIONS] VARIABLE_ID -With this command you can update the value or the template, as well as the description of a project variable. +With this command you can update the value or the template, as well as the description and the sensitive flag of a variable. !!! note If you update the template of a static variable, it will be transformed to a template based variable. If you want to change the value of a template based variable, an error will be shown. +!!! note + The value and the template of a sensitive variable can not be read. An update which does not set them keeps the stored ones. Since it is not known whether a sensitive variable is template based, a new value or template replaces whatever it had. Removing the sensitive flag with `--no-sensitive` needs a new `--value` or `--template`. + + ??? info "Options" ```text - --value TEXT The new value of the project variable. - --template TEXT The new template of the project variable. You can use - Jinja template syntax, e.g. use '{{global.myVar}}' for - accessing global variables, or '{{project.myVar}}' for - accessing variables from the same project. - --description TEXT The new description of the project variable. + --value TEXT The new value of the project variable. + --template TEXT The new template of the project variable. You + can use Jinja template syntax, e.g. use + '{{global.myVar}}' for accessing global + variables, or '{{project.myVar}}' for + accessing variables from the same project. + --description TEXT The new description of the project variable. + --sensitive / --no-sensitive Flag the variable as sensitive, or remove the + flag. Without this option, the flag stays as + it is. Removing the flag needs a new value or + template. ``` diff --git a/docs/automate/cmemc-command-line-interface/command-reference/workflow/index.md b/docs/automate/cmemc-command-line-interface/command-reference/workflow/index.md index 29cffc150..885ba3afd 100644 --- a/docs/automate/cmemc-command-line-interface/command-reference/workflow/index.md +++ b/docs/automate/cmemc-command-line-interface/command-reference/workflow/index.md @@ -33,6 +33,12 @@ Executing a workflow can be done in two ways: Without `--wait` just sends the st The optional `--wait` option starts the workflows in the same way, but also polls the status of a workflow until it is finished. In case of an error of a workflow, the next workflow is not started. +Use `--parameter` or `--parameter-file` to override the default values of execution variables for this run only. The overrides are applied to every given workflow, and a warning is shown for each workflow which does not use or define a given name. A workflow is not started as long as an execution variable it requires has no value. A run with overrides is not shown as the default execution of the workflow: its status is listed under the activity ExecuteWorkflowWithPayload. With `--wait`, the finished run is removed again. + +!!! note + Workflows started by a scheduler can not take such overrides, they always use the default values of their execution variables. + + ??? info "Options" @@ -47,6 +53,16 @@ The optional `--wait` option starts the workflows in the same way, but also poll polls. Status polls are cheap, so a higher polling interval is most likely not needed. [default: 1; 0<=x<=60] + -p, --parameter ... Override the default value of an execution + variable for this run only. Pairs of + variable name and value need to be given as + a tuple 'NAME VALUE'. The option can be + given multiple times. The stored default + stays unchanged. + --parameter-file FILE Read execution variable overrides from a + JSON file with an object of variable names + and values. A name given with --parameter + wins over the same name in the file. ``` ## workflow io @@ -66,6 +82,8 @@ With this command, you can execute a workflow that uses replaceable datasets as Regarding the input dataset configuration - the following rules apply: If autoconfig is enabled ('--autoconfig', the default), the dataset configuration is guessed. If autoconfig is disabled ('--no-autoconfig') and the type of the dataset file is the same as the replaceable dataset in the workflow, the configuration from this dataset is copied. If autoconfig is disabled and the type of the dataset file is different from the replaceable dataset in the workflow, the default config is used. +Use `--parameter` or `--parameter-file` to override the default values of execution variables for this run only. The overrides are never part of the input data. The workflow is not started as long as an execution variable it requires has no value. + ??? info "Options" @@ -96,6 +114,16 @@ With this command, you can execute a workflow that uses replaceable datasets as e.g. in order to process CSV files with semicolon- instead of comma-separation. [default: autoconfig] + -p, --parameter ... Override the default value of an execution + variable for this run only. Pairs of + variable name and value need to be given as + a tuple 'NAME VALUE'. The option can be + given multiple times. The stored default + stays unchanged. + --parameter-file FILE Read execution variable overrides from a + JSON file with an object of variable names + and values. A name given with --parameter + wins over the same name in the file. ``` ## workflow list @@ -152,6 +180,34 @@ cmemc workflow status [OPTIONS] [WORKFLOW_IDS]... ExecuteWorkflowWithPayload ``` +## workflow variables + +List the execution variables a workflow run needs. + +```shell-session title="Usage" +cmemc workflow variables [OPTIONS] WORKFLOW_ID +``` + + + + +Outputs the execution variables which can or have to be given when the workflow is executed, e.g. with the ``--parameter`` option of the execute and io commands. This covers the variables used by the tasks of the workflow and of its sub-workflows, and the variables defined on the workflow itself. + +A variable is required if it is used during the run, but the workflow defines no default value for it and no preceding task sets it. A workflow is not started as long as a required variable has no value. Note that only the defaults defined on the workflow itself apply to a run, the defaults of its tasks do not. + +The table states for each variable the tasks which use it and the tasks which set it during the run. + + + +??? info "Options" + ```text + + --raw Outputs raw JSON. + --id-only Lists only the names of the execution variables and no other + metadata. This is useful for piping the names into other + commands. + ``` + ## workflow open Open a workflow in your browser. diff --git a/docs/automate/cmemc-command-line-interface/configuration/certificate-handling-and-ssl-verification/index.md b/docs/automate/cmemc-command-line-interface/configuration/certificate-handling-and-ssl-verification/index.md index 90427c283..00ee50689 100644 --- a/docs/automate/cmemc-command-line-interface/configuration/certificate-handling-and-ssl-verification/index.md +++ b/docs/automate/cmemc-command-line-interface/configuration/certificate-handling-and-ssl-verification/index.md @@ -10,14 +10,14 @@ tags: ## Introduction -In a reasonable production deployment, all client-accessible Corporate Memory APIs will be securely available as HTTPS endpoints. +In a reasonable production deployment, all client-accessible eccenca Corporate Memory APIs will be securely available as HTTPS endpoints. This document clarifies how to deal with certificates. cmemc will validate the certificates of your HTTPS endpoints and indicate validation errors. If the certificates of your Corporate Memory deployment are based on a common and publicly available Certificate Authority (such as [Let's Encrypt](https://letsencrypt.org/)), cmemc is able to validate your certificates out of the box. However, in some cases you need to do one of the following: -- provide your own certificates CA bundle in order to allow cmem to trust your servers +- provide your own certificates CA bundle in order to allow cmemc to trust your servers - disable SSL verification entirely (for debugging and testing purpose only) ## Provide your own CA bundle diff --git a/docs/automate/cmemc-command-line-interface/configuration/completion-setup/index.md b/docs/automate/cmemc-command-line-interface/configuration/completion-setup/index.md index 1089a92cf..efc5787f6 100644 --- a/docs/automate/cmemc-command-line-interface/configuration/completion-setup/index.md +++ b/docs/automate/cmemc-command-line-interface/configuration/completion-setup/index.md @@ -12,7 +12,7 @@ tags: In case you are using bash or zsh as your terminal shell, you should enable [Command-line tab completion](https://en.wikipedia.org/wiki/Command-line_completion) for cmemc. Tab completion is a powerful feature and will save you a lot of typing work. -Furthermore, it will help you to learn the different commands, parameters and options and will auto-complete parameter values taken live from your Corporate Memory instance (such as graph IRIs, project IDs, etc.). +Furthermore, it will help you to learn the different commands, parameters and options and will auto-complete parameter values taken live from your eccenca Corporate Memory instance (such as graph IRIs, project IDs, etc.). ![cmemc - Create Dataset](22.1-cmemc-create-dataset.gif "cmemc - Create Dataset") diff --git a/docs/automate/cmemc-command-line-interface/configuration/environment-based-configuration/index.md b/docs/automate/cmemc-command-line-interface/configuration/environment-based-configuration/index.md index 7c8add5a3..6c0eef4d8 100644 --- a/docs/automate/cmemc-command-line-interface/configuration/environment-based-configuration/index.md +++ b/docs/automate/cmemc-command-line-interface/configuration/environment-based-configuration/index.md @@ -69,7 +69,7 @@ export CMEMC_CONFIG_FILE=cmemc.ini export CMEMC_CONNECTION=mycmem ``` -This alone allows us to save a lot of typing for a series of commands on the same Corporate Memory instance. +This alone allows us to save a lot of typing for a series of commands on the same eccenca Corporate Memory instance. ``` shell-session $ cmemc graph list --raw diff --git a/docs/automate/cmemc-command-line-interface/configuration/file-based-configuration/index.md b/docs/automate/cmemc-command-line-interface/configuration/file-based-configuration/index.md index 3427d9e7a..4f80244fa 100644 --- a/docs/automate/cmemc-command-line-interface/configuration/file-based-configuration/index.md +++ b/docs/automate/cmemc-command-line-interface/configuration/file-based-configuration/index.md @@ -31,7 +31,7 @@ If you start cmemc without any command or option, it outputs a general introduct $ cmemc Usage: cmemc [OPTIONS] COMMAND [ARGS]... - Eccenca Corporate Memory Control (cmemc). + eccenca Corporate Memory Control (cmemc). cmemc is the eccenca Corporate Memory Command Line Interface (CLI). @@ -88,7 +88,7 @@ Empty config created: config.ini No connections found. ``` -You can now edit your configuration file and add credentials and URL parameters for your Corporate Memory deployment. +You can now edit your configuration file and add credentials and URL parameters for your eccenca Corporate Memory deployment. You either search for the configuration manually in your home directory or you can use the `config edit` command, which opens the configuration file in your default text editor (specified by the [`EDITOR` variable](https://wiki.archlinux.org/title/environment_variables#Default_programs)). ``` shell-session diff --git a/docs/automate/cmemc-command-line-interface/configuration/getting-credentials-from-external-processes/index.md b/docs/automate/cmemc-command-line-interface/configuration/getting-credentials-from-external-processes/index.md index 349abd9a2..034040c42 100644 --- a/docs/automate/cmemc-command-line-interface/configuration/getting-credentials-from-external-processes/index.md +++ b/docs/automate/cmemc-command-line-interface/configuration/getting-credentials-from-external-processes/index.md @@ -54,7 +54,7 @@ export OAUTH_CLIENT_SECRET=$(get-my-secret.sh) ## External Processes -Another option, which is interesting when working with multiple Corporate Memory instances, is the configuration of an external process in your [cmemc configuration file](../file-based-configuration/index.md). +Another option, which is interesting when working with multiple eccenca Corporate Memory instances, is the configuration of an external process in your [cmemc configuration file](../file-based-configuration/index.md). In order to get credential information from an external process you need to use the following configuration variables to set up an external executable: diff --git a/docs/automate/cmemc-command-line-interface/finding-and-filtering-objects/index.md b/docs/automate/cmemc-command-line-interface/finding-and-filtering-objects/index.md index b34dfd2b8..aee87c37d 100644 --- a/docs/automate/cmemc-command-line-interface/finding-and-filtering-objects/index.md +++ b/docs/automate/cmemc-command-line-interface/finding-and-filtering-objects/index.md @@ -58,7 +58,7 @@ The following table gives an impression of how different these name sets are: !!! tip With [command-line completion](../configuration/completion-setup/index.md) enabled, you do not need to remember any of this. - Pressing ++tab++ after `--filter` offers the filter names of the current command, and pressing ++tab++ again often completes the values as well, taken live from your Corporate Memory instance. + Pressing ++tab++ after `--filter` offers the filter names of the current command, and pressing ++tab++ again often completes the values as well, taken live from your eccenca Corporate Memory instance. ## Combining filters diff --git a/docs/automate/cmemc-command-line-interface/scripting-with-cmemc/index.md b/docs/automate/cmemc-command-line-interface/scripting-with-cmemc/index.md index eaf62d9f0..19fbd412c 100644 --- a/docs/automate/cmemc-command-line-interface/scripting-with-cmemc/index.md +++ b/docs/automate/cmemc-command-line-interface/scripting-with-cmemc/index.md @@ -49,7 +49,7 @@ Combining `--id-only` with `--filter` is the basic building block of most cmemc ## JSON output -The `--raw` option is available on more than 40 commands and outputs the JSON as it was received from the Corporate Memory APIs. +The `--raw` option is available on more than 40 commands and outputs the JSON as it was received from the eccenca Corporate Memory APIs. Together with a tool such as [jq](https://jqlang.github.io/jq/), this allows for arbitrary post-processing: ``` shell-session title="IRIs of all writeable graphs" diff --git a/docs/automate/cmemc-command-line-interface/troubleshooting-and-caveats/index.md b/docs/automate/cmemc-command-line-interface/troubleshooting-and-caveats/index.md index 6622577f1..8bcc64c40 100644 --- a/docs/automate/cmemc-command-line-interface/troubleshooting-and-caveats/index.md +++ b/docs/automate/cmemc-command-line-interface/troubleshooting-and-caveats/index.md @@ -11,7 +11,7 @@ This page lists and documents possible issues and warnings when working with cme ## Proxy is in the way -If you feel that your system's proxy configuration negatively impacts the communication between cmemc and Corporate Memory, you can disable using any proxy by setting this variable: +If you feel that your system's proxy configuration negatively impacts the communication between cmemc and eccenca Corporate Memory, you can disable using any proxy by setting this variable: ``` shell-session export no_proxy='*' diff --git a/docs/automate/continuous-integration/index.md b/docs/automate/continuous-integration/index.md index c7780dc01..6e3b9b987 100644 --- a/docs/automate/continuous-integration/index.md +++ b/docs/automate/continuous-integration/index.md @@ -7,7 +7,7 @@ subtitle: and Delivery ## Introduction Project teams often manage crucial parts of their work assets inside of git repositories. -This includes Corporate Memory related files such as ontologies, shapes or project configurations. +This includes eccenca Corporate Memory related files such as ontologies, shapes or project configurations. Given such a project setup, it is often wanted to start activities with these Corporate Memory files. Continuous integration (CI) is the practice of automating the integration of changes from multiple contributors into a single project. Originated from software projects, CI can (and should) be applied to Knowledge Graph projects as well. diff --git a/docs/automate/processing-data-with-variable-input-workflows/index.md b/docs/automate/processing-data-with-variable-input-workflows/index.md index 169067499..bef666841 100644 --- a/docs/automate/processing-data-with-variable-input-workflows/index.md +++ b/docs/automate/processing-data-with-variable-input-workflows/index.md @@ -10,7 +10,7 @@ tags: ## Introduction -This tutorial shows how you can create and use data integration workflows to process data coming from outside Corporate Memory (i.e., without registering datasets). +This tutorial shows how you can create and use data integration workflows to process data coming from outside eccenca Corporate Memory (i.e., without registering datasets). This is achieved with the **Allow replacement** flag on input and/or output datasets. This flag is supported by most file dataset types. A replaceable dataset is created and used inside a workflow as an input for other tasks (e.g., a transformation). diff --git a/docs/automate/scheduling-workflows/index.md b/docs/automate/scheduling-workflows/index.md index 7c35df02d..a30e36caa 100644 --- a/docs/automate/scheduling-workflows/index.md +++ b/docs/automate/scheduling-workflows/index.md @@ -9,7 +9,7 @@ tags: ## Introduction -For a time-based execution of a workflow, Corporate Memory provides the Scheduler operator. Please note that, in case you want to schedule workflows externally, [cmemc can be used for that](../cmemc-command-line-interface/workflow-execution-and-orchestration/index.md). +For a time-based execution of a workflow, eccenca Corporate Memory provides the Scheduler operator. Please note that, in case you want to schedule workflows externally, [cmemc can be used for that](../cmemc-command-line-interface/workflow-execution-and-orchestration/index.md). ## Create a scheduler diff --git a/docs/build/active-learning/index.md b/docs/build/active-learning/index.md index f33b3ca7d..2ccac6342 100644 --- a/docs/build/active-learning/index.md +++ b/docs/build/active-learning/index.md @@ -15,7 +15,7 @@ Active learning infuses expert knowledge and creates new relationships between p > Active learning is a special case of machine learning in which a learning algorithm interactively queries a user to label new data points with the desired outputs. [[wikipedia]](https://www.wikiwand.com/en/Active_learning_(machine_learning)#introduction) -In Corporate Memory we apply this approach to the process of learning a linking rule by interactively label records from the configured source and target dataset. +In eccenca Corporate Memory we apply this approach to the process of learning a linking rule by interactively label records from the configured source and target dataset. Labeling in this case means to indicate if the pair of resources (from source and target) should be connected with the configured property. The labeling process creates reference links against which the linking rule can be created and further refined as more input is given by the user. diff --git a/docs/build/cool-iris/index.md b/docs/build/cool-iris/index.md index 2cc0fff4f..03fcee214 100644 --- a/docs/build/cool-iris/index.md +++ b/docs/build/cool-iris/index.md @@ -28,7 +28,7 @@ The following list of example IRIs demonstrate the broad scope of this concept: - `telnet://192.0.2.16:80/` - `urn:oasis:names:specification:docbook:dtd:xml:4.1.2` -## Best practices in Corporate Memory +## Best practices in eccenca Corporate Memory A good IRI is unique, stable, simple and manageable. diff --git a/docs/build/define-prefixes-namespaces/index.md b/docs/build/define-prefixes-namespaces/index.md index 4d000922e..a6c05d286 100644 --- a/docs/build/define-prefixes-namespaces/index.md +++ b/docs/build/define-prefixes-namespaces/index.md @@ -39,7 +39,7 @@ In the Explore area, an Ontology with a correct namespace declaration looks like ### Using cmemc -The `vocabulary` command group of [cmemc](../../automate/cmemc-command-line-interface/index.md) has an `import` command that you can use to install arbitrary vocabulary documents and register them as vocabularies in Corporate Memory. +The `vocabulary` command group of [cmemc](../../automate/cmemc-command-line-interface/index.md) has an `import` command that you can use to install arbitrary vocabulary documents and register them as vocabularies in eccenca Corporate Memory. Beginning with v22.2, this command has an additional option `--namespace` which you can use to set a vocabulary namespace even if the vocabulary does not include the data needed for autodiscovery: diff --git a/docs/build/evaluate-template/index.md b/docs/build/evaluate-template/index.md index c0a22ca92..63c5b8341 100644 --- a/docs/build/evaluate-template/index.md +++ b/docs/build/evaluate-template/index.md @@ -56,7 +56,7 @@ The tutorial consists of the following steps, which are described in detail belo The vocabulary contains the classes and properties needed to map the source data into entities in the Knowledge Graph. -1. In Corporate Memory, click **Knowledge Graphs** in the navigation under **Explore** on the left side of the page. +1. In eccenca Corporate Memory, click **Knowledge Graphs** in the navigation under **Explore** on the left side of the page. ![Menu entry EXPLORE > Knowledge Graphs](menu-explore-knowledge-graphs.png){ class="bordered" width="40%" } diff --git a/docs/build/extracting-data-from-a-web-api/index.md b/docs/build/extracting-data-from-a-web-api/index.md index 54e7005af..02d937b18 100644 --- a/docs/build/extracting-data-from-a-web-api/index.md +++ b/docs/build/extracting-data-from-a-web-api/index.md @@ -91,7 +91,7 @@ As we are only interested in the _HTTP Message Body_ which holds the JSON reposi ## 3 Create a JSON Dataset -To create a JSON-to-RDF-mapping within Corporate Memory, we have to first register an example response from the API (repos.json). Based on the schema of the response, we can then define step-by-step the mappings, which are used to build the Knowledge Graph. +To create a JSON-to-RDF-mapping within eccenca Corporate Memory, we have to first register an example response from the API (repos.json). Based on the schema of the response, we can then define step-by-step the mappings, which are used to build the Knowledge Graph. 1. Click the **Create** button (top right) in the data integration workspace and select the type **JSON**. diff --git a/docs/build/index.md b/docs/build/index.md index e237f7a1d..79ba3ffbb 100644 --- a/docs/build/index.md +++ b/docs/build/index.md @@ -63,6 +63,6 @@ The Build stage turns source data from files, databases, APIs, and streams into Look up tasks, operators and supported services. - [Task and Operator Reference](reference/index.md) --- Tasks and operators of the **Build** workspace. - - [Integrations](integrations/index.md) --- Services and applications that integrate with Corporate Memory. + - [Integrations](integrations/index.md) --- Services and applications that integrate with eccenca Corporate Memory. diff --git a/docs/build/integrations/index.md b/docs/build/integrations/index.md index 9dbd48663..5071b787d 100644 --- a/docs/build/integrations/index.md +++ b/docs/build/integrations/index.md @@ -181,7 +181,7 @@ geocoding service, such as [Nominatim](https://nominatim.org/), --- - Execute a [GraphQL query](../../build/reference/customtask/cmem_plugin_graphql-workflow-graphql-GraphQLPlugin.md) and process the result in a workflow. + Execute a [GraphQL query](../../build/reference/customtask/cmem_plugin_graphql-Query.md) and process the result in a workflow. - :simple-apachehive:{ .lg .middle } Hive diff --git a/docs/build/introduction-to-the-user-interface/index.md b/docs/build/introduction-to-the-user-interface/index.md index 81fff3438..2f1c8ec12 100644 --- a/docs/build/introduction-to-the-user-interface/index.md +++ b/docs/build/introduction-to-the-user-interface/index.md @@ -59,7 +59,7 @@ A dataset represents an abstraction over raw data. In order to work with data, y There is a range of different dataset types for different kinds of source data. Important dataset types include: -- **Knowledge Graph** - Read RDF from or write RDF to a Knowledge Graph embedded in Corporate Memory. +- **Knowledge Graph** - Read RDF from or write RDF to a Knowledge Graph embedded in eccenca Corporate Memory. - **CSV** - Read from or write to an CSV file. - **XML** - Read from or write to an XML file. - **JSON** - Read from or write to a JSON file. diff --git a/docs/build/kafka-consumer/index.md b/docs/build/kafka-consumer/index.md index 907a6abe5..9186585e2 100644 --- a/docs/build/kafka-consumer/index.md +++ b/docs/build/kafka-consumer/index.md @@ -13,7 +13,7 @@ tags: [Apache Kafka](https://kafka.apache.org/) is a distributed event store and stream-processing platform. Kafka is widely used in enterprises for data pipelines, streaming analytics, data integration and other applications. -By using the [cmem-plugin-kafka](https://pypi.org/project/cmem-plugin-kafka/) [Python Plugin](../../develop/python-plugins/index.md), you can consume messages from Apache Kafka and use them inside Corporate Memory Workflows. +By using the [cmem-plugin-kafka](https://pypi.org/project/cmem-plugin-kafka/) [Python Plugin](../../develop/python-plugins/index.md), you can consume messages from Apache Kafka and use them inside eccenca Corporate Memory Workflows. ## Installation diff --git a/docs/build/lift-data-from-tabular-data-such-as-csv-xslx-or-database-tables/index.md b/docs/build/lift-data-from-tabular-data-such-as-csv-xslx-or-database-tables/index.md index d212a61c6..e430011a7 100644 --- a/docs/build/lift-data-from-tabular-data-such-as-csv-xslx-or-database-tables/index.md +++ b/docs/build/lift-data-from-tabular-data-such-as-csv-xslx-or-database-tables/index.md @@ -120,7 +120,7 @@ The vocabulary contains the classes and properties needed to map the data into t === "JDBC" - Instead of uploading the [services.csv](services.csv) sample file into Corporate Memory, you can also load it into a SQL database and access it from Corporate Memory using the JDBC protocol. + Instead of uploading the [services.csv](services.csv) sample file into eccenca Corporate Memory, you can also load it into a SQL database and access it from Corporate Memory using the JDBC protocol. 1. Click again on **Create new** :eccenca-item-add-artefact: in the top right corner. In the **Create new item** window, select **JDBC endpoint** type and click **Add**. diff --git a/docs/build/loading-jdbc-datasets-incrementally/index.md b/docs/build/loading-jdbc-datasets-incrementally/index.md index bd38907f6..1dfcd50f9 100644 --- a/docs/build/loading-jdbc-datasets-incrementally/index.md +++ b/docs/build/loading-jdbc-datasets-incrementally/index.md @@ -33,7 +33,7 @@ This tutorial walks you through the process of loading data incrementally from a ## 1 Create the JDBC dataset -To extract data from a relational database, you need to first register a **JDBC endpoint** in Corporate Memory. This tutorial assumes that you have access to the relational database from the Corporate Memory instance. +To extract data from a relational database, you need to first register a **JDBC endpoint** in eccenca Corporate Memory. This tutorial assumes that you have access to the relational database from the Corporate Memory instance. 1. In Corporate Memory, click Projects under **BUILD** in the navigation on the left side of the page. @@ -82,13 +82,13 @@ Now that we have the JDBC endpoint IRI, we will build the Metadata Graph to stor The following three RDF triples hold the (minimal) necessary information we need for this tutorial: -1. The first triple imports the CMEM DI Project graph into our Metadata Graph to enable access to the LIMIT property defined earlier and to additional information we may need in the future. +1. The first triple imports the **CMEM DI Project** graph into our Metadata Graph to enable access to the LIMIT property defined earlier and to additional information we may need in the future. 2. The second triple defines a label for the Graph. 3. The third triple defines the <...**lastOffset**> property we need for this tutorial. As a default, we set it to 0 to start with the first row in the table. For your project: -1. adjust the CMEM DI Project IRI and +1. adjust the **CMEM DI Project** IRI and 2. the JDBC endpoint IRI. **Import the Graph** in the Exploration tab → Graph (menu) → Add new Graph → Provide Graph IRI + Select file. diff --git a/docs/build/reference/customtask/.pages b/docs/build/reference/customtask/.pages index e347f8a6f..b14def45f 100644 --- a/docs/build/reference/customtask/.pages +++ b/docs/build/reference/customtask/.pages @@ -18,9 +18,11 @@ nav: - "Download Git files": cmem_plugin_git-Download.md - "Download Nextcloud files": cmem_plugin_nextcloud-Download.md - "Download Office 365 Files": cmem_plugin_office365-Download.md + - "Download S3 objects": cmem_plugin_s3-Download.md - "Download SSH files": cmem_plugin_ssh-Download.md - "Evaluate template": Template.md - "Execute a command in a kubernetes pod": cmem_plugin_kubernetes-Execute.md + - "Execute Catalog Query": cmem_plugin_querycatalog-ExecuteCatalogQuery.md - "Execute commands via SSH": cmem_plugin_ssh-Execute.md - "Execute Instructions": cmem_plugin_llm-ExecuteInstructions.md - "Execute REST requests": eccencaRestOperator.md @@ -29,10 +31,15 @@ nav: - "Generate base36 IRDIs": cmem_plugin_irdi-workflow-irdi_plugin-IrdiPlugin.md - "Generate random values": cmem_plugin_random-GenerateEntities.md - "Generate SHACL shapes from data": cmem_plugin_shapes-plugin_shapes-ShapesPlugin.md + - "Get Catalog Query": cmem_plugin_querycatalog-GetCatalogQuery.md - "Get emails": cmem-plugin-email-GetEmails.md + - "Get ntfy attachments": cmem_plugin_ntfy-GetAttachments.md + - "Get ntfy messages": cmem_plugin_ntfy-GetMessages.md - "Get project files": getProjectFiles.md - "Get workflow report": cmem_plugin_wfreports_get_report.md - - "GraphQL query": cmem_plugin_graphql-workflow-graphql-GraphQLPlugin.md + - "GraphQL query": cmem_plugin_graphql-Query.md + - "Hetzner Cloud: List Resources": cmem_plugin_hetzner-ListResources.md + - "Hetzner Cloud: List Server Types": cmem_plugin_hetzner-ListServerTypes.md - "Install Packages": cmem_plugin_packages-InstallPackages.md - "Join tables": Merge.md - "jq": cmem-plugin-jq-workflow.md @@ -40,10 +47,12 @@ nav: - "JSON to File": jsonToFile.md - "Kafka Consumer (Receive Messages)": cmem_plugin_kafka-ReceiveMessages.md - "Kafka Producer (Send Messages)": cmem_plugin_kafka-SendMessages.md + - "List Catalog Queries": cmem_plugin_querycatalog-ListCatalogQueries.md - "List Git files": cmem_plugin_git-List.md - "List Nextcloud files": cmem_plugin_nextcloud-List.md - "List Office 365 Files": cmem_plugin_office365-List.md - "List project files": cmem_plugin_project_resources-List.md + - "List S3 objects": cmem_plugin_s3-List.md - "List SSH files": cmem_plugin_ssh-List.md - "Merge tables": MultiTableMerge.md - "Normalize units of measurement": ucumNormalizationTask.md @@ -60,10 +69,13 @@ nav: - "Scheduler": Scheduler.md - "Search addresses": SearchAddresses.md - "Search for Logs": cmem_plugin_logpoint-search_logs_task-RetrieveLogs.md + - "Search LDAP directory": cmem_plugin_ldap-Search.md - "Search Vector Embeddings": cmem_plugin_pgvector-Search.md - "Select random entities": cmem_plugin_random-SelectEntities.md - "Send email": SendEMail.md - "Send Mattermost messages": cmem_plugin_mattermost.md + - "Send ntfy attachment": cmem_plugin_ntfy-SendAttachment.md + - "Send ntfy message": cmem_plugin_ntfy-SendMessage.md - "Set execution variable": setExecutionVariableOperator.md - "Set or Overwrite parameters": cmem_plugin_parameters-ParametersPlugin.md - "Set parameters": setParameters.md @@ -82,6 +94,7 @@ nav: - "Upload File to Knowledge Graph": eccencaDataPlatformGraphStoreFileUploadOperator.md - "Upload files to CKAN": cmem_plugin_ckan-Upload.md - "Upload files to Nextcloud": cmem_plugin_nextcloud-Upload.md + - "Upload files to S3": cmem_plugin_s3-Upload.md - "Upload Git files": cmem_plugin_git-Upload.md - "Upload local files": cmem_plugin_project_resources-UploadLocalFiles.md - "Upload SSH files": cmem_plugin_ssh-Upload.md diff --git a/docs/build/reference/customtask/CustomSQLExecution.md b/docs/build/reference/customtask/CustomSQLExecution.md index bdff0f419..e51064b9d 100644 --- a/docs/build/reference/customtask/CustomSQLExecution.md +++ b/docs/build/reference/customtask/CustomSQLExecution.md @@ -1,6 +1,6 @@ --- title: "Spark SQL query" -description: "Executes a custom SQL query on the first input Spark dataframe and returns the result as its output." +description: "Executes a Spark SQL query on the input entities and returns the result as its output." icon: octicons/cross-reference-24 tags: - WorkflowTask @@ -12,14 +12,59 @@ tags: -Executes a custom SQL query on the first input Spark dataframe and returns the result as its output. +The Spark SQL query operator runs a [Spark SQL](https://spark.apache.org/docs/latest/sql-ref.html) query against the +entities of its single input and passes the query result to the next workflow task. + +The operator requires the Spark execution backend (`execution.manager.plugin = SparkExecutionManager`). In local +workflow execution the task fails because no executor is available for it. + +## Command + + +The **Command** parameter holds the SQL query. The input entities are made available as a table named `dataset`, +regardless of which task is connected to the input. Each column holds all values of its path as an array of strings, +since an entity can have several values per path. Use `try_element_at(column, 1)` to read the first value, which +returns NULL if the entity has no value, and cast it to compare it as a number. For example, this query keeps the +entities whose first `age` value is above 30 and returns two of their columns: + +```sql +SELECT try_element_at(name, 1) AS name, try_element_at(age, 1) AS age +FROM dataset +WHERE try_cast(try_element_at(age, 1) AS INT) > 30 +``` + +The columns hold plain strings instead of arrays if the configuration option `spark.interpreter.options.columnType` +is set to any value other than `sequence`. + +The columns of the `dataset` table are named after the paths of the input schema: the column headers of a CSV file or +the full property URIs of RDF data. Enclose a column name in backticks when it contains characters that are not valid +in an SQL identifier, such as a URI: + +```sql +SELECT `http://xmlns.com/foaf/0.1/name` AS name FROM dataset +``` + +The entity URIs of the input are not part of the table and cannot be referenced in the query. + +The table name is applied by a case-sensitive text replacement: every occurrence of the character sequence `dataset` in +the query is replaced with the internal name of the input table, also inside longer names such as `my_dataset` or inside +a URI, while `DATASET` or `Dataset` is left unchanged. Write the table name in lower case and make sure that column names, +aliases and string literals in the query do not contain `dataset`. + +## Output + +The columns of the query result become the paths of the output schema, in the order the query returns them. A column +name that is not valid in a URI, e.g. an alias containing spaces, is URL-encoded in the path, so prefer plain +alphanumeric aliases. The output schema declares all result values as strings and keeps the type of the input entities. +Connect a dataset or another operator to the output to use the result. + ## Parameter ### Command -SQL command. The name of the table in the statement must be 'dataset', regardless the input. +SQL command. The name of the table in the statement must be 'dataset', regardless of the input. - ID: `command` - Datatype: `code-sql` diff --git a/docs/build/reference/customtask/SendEMail.md b/docs/build/reference/customtask/SendEMail.md index 14aba4cfb..c5dccc2ff 100644 --- a/docs/build/reference/customtask/SendEMail.md +++ b/docs/build/reference/customtask/SendEMail.md @@ -28,6 +28,12 @@ Sends an email using an SMTP server with support for both plain text and HTML fo When connected to a dataset that is based on a file in a workflow, it will send that file as an attachment whenever the workflow is executed. This allows you to send workflow results via email automatically. +## SSL/TLS + +The connection is upgraded with STARTTLS if the server offers it. "Force SSL" uses SSL/TLS from the start. +The certificate of the server must be trusted by the Java runtime and match the configured host. +The host name check can be switched off with the advanced parameter "Verify server host name". + ## HTML email example Enable HTML formatting and use standard HTML markup in your message: @@ -229,3 +235,13 @@ The number of retries per email when send errors are encountered. - ID: `nrRetries` - Datatype: `int` - Default Value: `2` + + + +### Verify server host name + +When enabled, the certificate of the SMTP server must match the configured host for SSL/TLS connections. Disable this only if the server cannot present a matching certificate, because the connection can then be intercepted. + +- ID: `verifyServerHostName` +- Datatype: `boolean` +- Default Value: `true` diff --git a/docs/build/reference/customtask/Template.md b/docs/build/reference/customtask/Template.md index 136105835..4e338f2f1 100644 --- a/docs/build/reference/customtask/Template.md +++ b/docs/build/reference/customtask/Template.md @@ -1,6 +1,6 @@ --- title: "Evaluate template" -description: "Evaluates a template on a sequence of entities. Can be used after a transformation or directly after datasets that output a single table, such as CSV or Excel." +description: "Evaluates a template on a sequence of entities. Can be used after a transformation or directly after datasets that output a single table, such as CSV or Excel. A template that uses no input variables can be evaluated without an input." icon: octicons/cross-reference-24 tags: - WorkflowTask @@ -17,6 +17,7 @@ The template operator supports the Jinja templating language. Documentation abou Note that support for RDF properties is limited, because Jinja does not support some special characters (in particula colons) in variable names. This makes it impractical to access RDF properties. For this reason, the transformation that precedes the template operator needs to make sure that it generates attributes that are valid Jinja variable names. + ## Default evaluation By default, the template is evaluated separately for each entity. @@ -38,7 +39,7 @@ A single output entity will be generated that contains the evaluated template. If the input entities are hierarchical (typically the case if the input transformation is hierarchical), each entity will be hierarchical as well. -Example iterating over an sequence of books that each contains a list of chapters: +Example iterating over a sequence of books that each contains a list of chapters: {% for book in entities %} Book {{book.title}} @@ -49,6 +50,15 @@ Example iterating over an sequence of books that each contains a list of chapter In this example, the child mapping defines a `chapter` target property from which it is accessible from the root entities. If the child mapping allows multiple entities, the value of the property will be a list of entities. +## Static evaluation + +The template operator can be used without any inputs. The input port will only appear if a variable referencing an input is used or if the 'Forward input attributes' option is enabled. +E.g. the use of the 'entities' variable in case of the full evaluation would require the connection of an input. + +Variables with the scopes 'execution', 'project' and 'global' can be used without the need for an input task/dataset. + +Without an input port, the template is evaluated once and a single output entity is generated. An input task/dataset that is still connected to the operator is not read. + ## Parameter diff --git a/docs/build/reference/customtask/cmem-plugin-jq-workflow.md b/docs/build/reference/customtask/cmem-plugin-jq-workflow.md index 83f4d1425..abe7dbdfc 100644 --- a/docs/build/reference/customtask/cmem-plugin-jq-workflow.md +++ b/docs/build/reference/customtask/cmem-plugin-jq-workflow.md @@ -18,6 +18,27 @@ tags: e.g. with cmemc. +This task applies a jq filter to a JSON file and writes the transformed +result to a new JSON file. + +Both the input and the output port carry a single file per entity. The input +file is read as JSON, transparently decompressing it first if it is +gzip-compressed. The output file is always plain (uncompressed) JSON, encoded +as UTF-8 and pretty-printed with two-space indentation. + +The task is typically placed between two file-based tasks, for example +reading from a **JSON dataset** and writing to another one, so that the +filter output becomes the next task's input. + +If the filter produces exactly one JSON value for a file, that value becomes +the file's entire content. If it produces zero or several values, the output +file instead contains a JSON array of all of them - so the shape of the +output can change from a single object to an array depending on how many +values the filter happens to emit. The whole input file is read into memory +before the filter runs, so very large files are not streamed. Cancelling the +containing workflow only takes effect between files, not while a file is +being transformed. + > [jq](https://jqlang.org/) is like sed for JSON data - you can use it to > slice and filter and map and transform structured data with the same ease that sed, awk, > grep and friends let you play with text. @@ -90,9 +111,9 @@ The jq program to apply to the input JSON string. ## Advanced Parameter -### JSON source which can be used with the validate expression action - +### Validation source +Example JSON used by the Validate expression action to preview the filter's output before running the task. Leave empty to skip the preview. - ID: `validation_source` - Datatype: `code-json` diff --git a/docs/build/reference/customtask/cmem_plugin_apify-RunActor.md b/docs/build/reference/customtask/cmem_plugin_apify-RunActor.md index 8081ffa92..7fe9a2f72 100644 --- a/docs/build/reference/customtask/cmem_plugin_apify-RunActor.md +++ b/docs/build/reference/customtask/cmem_plugin_apify-RunActor.md @@ -32,7 +32,7 @@ and renders a list where a list is supplied. The result leaves through the file port as one file per actor run, in the chosen format. It is deliberately not turned into entities here: a dataset can be very large, and reading structure out of a file is what the JSON, CSV and XML -datasets of Corporate Memory already do. +datasets of eccenca Corporate Memory already do. The task covers three cases, which change its shape. Reading an existing Apify dataset starts no run and needs no input. Running an actor without reading it diff --git a/docs/build/reference/customtask/cmem_plugin_graphql-Query.md b/docs/build/reference/customtask/cmem_plugin_graphql-Query.md new file mode 100644 index 000000000..9be4c7936 --- /dev/null +++ b/docs/build/reference/customtask/cmem_plugin_graphql-Query.md @@ -0,0 +1,121 @@ +--- +title: "GraphQL query" +description: "Sends a GraphQL query or mutation to an endpoint and returns the result as entities." +icon: octicons/cross-reference-24 +tags: + - WorkflowTask + - PythonPlugin +--- + +# GraphQL query + + + +!!! note inline end "Python Plugin" + + This operator is part of a Python Plugin Package. + In order to use it, you need to install it, + e.g. with cmemc. + +This task sends a GraphQL query or mutation to an endpoint and +captures the response. + +## How often the endpoint is called + +Jinja syntax in the query or in the variables turns the task into a loop: both are +rendered once per arriving entity and the endpoint is called once per entity, with a +failing entity logged, counted in the report and skipped rather than taking the whole +task down. Text without Jinja syntax is sent exactly once, and anything connected as +input is then ignored. + +## What leaves the task + +The responses leave on the output port in one of two shapes: + +- **entities**: one per call. Their paths are the fields the query asks for, under the + alias where a field has one, so the next task is offered the schema while the workflow + is drawn rather than only after a first run. A field that selects sub fields becomes a + relation, and the entities behind it follow the shape of the response. +- **file**: all responses of the run written to a single JSON file. What leaves the port + is that one file rather than the data in it. + +How many values a field carries is not part of a query - that lives in the endpoint's own +schema - so every path is offered as possibly multi valued and the entities are built to +match. A field the endpoint answers with a single object therefore arrives as a list +holding one entity, and a JSON dataset connected to the output port holds an array in +that place. + +## Where this task fits + +The task usually opens a chain: a GraphQL API at one end, and at the other a transform +that maps the response into a graph, or a dataset that keeps it for later steps. The file +shape suits the second kind of chain, and a task that uploads or stores what it is handed, +since a file travels through those without being taken apart on the way. + +## Caveats + +- **Three kinds of query describe nothing in advance**, and the entities then leave with a + schema that stays unknown until the task has run, which the next task has to accept as + it comes: one that does not parse as GraphQL on its own, which is the usual case for a + Jinja template because the placeholders sit where GraphQL expects values; one holding + more than one operation; and one whose top level is a fragment rather than plain fields. + A run that hands on a file is not affected, since a file is described the same way + whatever the query asks for. +- **Jinja text is never checked for GraphQL syntax errors until it is rendered**, so a + mistake in it surfaces while the task runs, as a failed entity, rather than as a + configuration error while the task is set up. + + +## Parameter + +### Endpoint + +The URL the query is sent to, for example `https://fruits-api.netlify.app/graphql`. A collective list of public GraphQL APIs is available [here](https://github.com/IvanGoncharov/graphql-apis). + +- ID: `graphql_url` +- Datatype: `string` +- Default Value: `None` + + + +### Access token + +The token the endpoint is authenticated with. It is sent as the `Authorization: Bearer ` header. Leave it empty for an endpoint that needs no authentication. For GitLab, a personal, project or group access token works, with scope `read_api` for queries or `api` for mutations. + +- ID: `access_token` +- Datatype: `password` +- Default Value: `None` + + + +### Output mode + +In which shape the responses leave this task. + +- ID: `output_mode` +- Datatype: `string` +- Default Value: `entities` + + + +### Query + +The query or mutation to send, written in [GraphQL](https://graphql.org/). May contain Jinja syntax, for example `{{ id }}`, which is rendered against each arriving entity before the query is sent. `query allFruits { fruits { id scientific_name fruit_name family climatic_zone } }` + +- ID: `graphql_query` +- Datatype: `multiline string` +- Default Value: `None` + + + +### Query variables + +The values for the variables the query declares, as a JSON object - `{"id": 1}` for a query taking `$id`. An empty object is sent for a query that declares none. May contain Jinja syntax, for example `{"id": {{ id }}}`, which is rendered against each arriving entity before the query is sent. + +- ID: `graphql_variable_values` +- Datatype: `multiline string` +- Default Value: `{}` + +## Advanced Parameter + +`None` diff --git a/docs/build/reference/customtask/cmem_plugin_graphql-workflow-graphql-GraphQLPlugin.md b/docs/build/reference/customtask/cmem_plugin_graphql-workflow-graphql-GraphQLPlugin.md deleted file mode 100644 index 57a41158e..000000000 --- a/docs/build/reference/customtask/cmem_plugin_graphql-workflow-graphql-GraphQLPlugin.md +++ /dev/null @@ -1,75 +0,0 @@ ---- -title: "GraphQL query" -description: "Executes a custom GraphQL query to a GraphQL endpoint and saves result to a JSON dataset." -icon: octicons/cross-reference-24 -tags: - - WorkflowTask - - PythonPlugin ---- - -# GraphQL query - - - -!!! note inline end "Python Plugin" - - This operator is part of a Python Plugin Package. - In order to use it, you need to install it, - e.g. with cmemc. - -This workflow task performs GraphQL operations by sending - queries, mutations, and variables over operations. Allows for customization - in the GraphQL query using, Jinja queries and Jinja variables, which can be - obtained from entities. The result of the query is saved as a JSON document - in a pre-created JSON dataset. - - -## Parameter - -### Endpoint - -The URL of the GraphQL endpoint you want to query. A collective list of public GraphQL APIs is available [here](https://github.com/IvanGoncharov/graphql-apis). Example Endpoint: `https://fruits-api.netlify.app/graphql` - -- ID: `graphql_url` -- Datatype: `string` -- Default Value: `None` - - - -### Query - -The query text of the GraphQL Query you want to execute. GraphQL is a query language for APIs and a runtime for fulfilling those queries with your existing data. Learn more on GraphQL [here](https://graphql.org/). Example Query: query allFruits { fruits { id scientific_name tree_name fruit_name family origin description climatic_zone } } - -- ID: `graphql_query` -- Datatype: `multiline string` -- Default Value: `None` - - - -### Query variables - -Pass dynamic variables when making a query or mutation. Example Variables: {"id" : 1} - -- ID: `graphql_variable_values` -- Datatype: `multiline string` -- Default Value: `{}` - -## Advanced Parameter - -### Target JSON Dataset - -The Dataset where this task will save the JSON results. - -- ID: `graphql_dataset` -- Datatype: `string` -- Default Value: `None` - - - -### OAuth access token - -Access token that connects to a GraphQL endpoint to authorize and secure user access to resources and data. - -- ID: `oauth_access_token` -- Datatype: `string` -- Default Value: `None` diff --git a/docs/build/reference/customtask/cmem_plugin_hetzner-ListResources.md b/docs/build/reference/customtask/cmem_plugin_hetzner-ListResources.md new file mode 100644 index 000000000..656466928 --- /dev/null +++ b/docs/build/reference/customtask/cmem_plugin_hetzner-ListResources.md @@ -0,0 +1,94 @@ +--- +title: "Hetzner Cloud: List Resources" +description: "Inventory all existing resources of a Hetzner Cloud project (servers, volumes, networks, load balancers, IPs, firewalls, SSH keys, snapshots, …), one entity per resource in a single union schema." +icon: octicons/cross-reference-24 +tags: + - WorkflowTask + - PythonPlugin +--- + +# Hetzner Cloud: List Resources + + + +!!! note inline end "Python Plugin" + + This operator is part of a Python Plugin Package. + In order to use it, you need to install it, + e.g. with cmemc. + + +This workflow operator inventories the resources actually provisioned in a +Hetzner Cloud project, using the [Hetzner Cloud API](https://docs.hetzner.cloud/). +It complements **Hetzner Cloud: List Server Types** (the catalog of what you +*could* create) by listing what you *have*. + +It produces one entity per resource across 11 resource types: `server`, +`volume`, `network`, `load_balancer`, `floating_ip`, `primary_ip`, `firewall`, +`ssh_key`, `image`, `certificate` and `placement_group`. All entities share a +single union schema; the `resourceType` column discriminates each row's type, +and each row fills its shared columns plus its own type's columns. A column +that does not apply to a row carries no value at all, rather than an empty one. + +Images are restricted to project-owned snapshots and backups; the shared +system / app OS catalog is excluded. Since the Hetzner API names only system +images, the `name` of a snapshot or backup falls back to its description. + +Note that `ipv6` holds the routed IPv6 network in CIDR notation for a server, +and a single address for the IP resources. + +## Authentication + +You need a Hetzner Cloud API token of a project (Console → Security → API +Tokens). A read-only token is sufficient. + +## Filters + +All filters are optional. Leave them empty to inventory everything. A filter +that matches nothing yields an empty result, which the execution report states. + +Located resources are the server, volume, primary IP, load balancer and +floating IP (by its home location). The network, firewall, SSH key, image, +certificate and placement group are global in the Hetzner API and always pass +the location filter. + + +## Parameter + +### API Token + +A Hetzner Cloud project API token (read-only is sufficient). + +- ID: `api_token` +- Datatype: `password` +- Default Value: `None` + +## Advanced Parameter + +### Resource type + +Restrict to a single resource type. Empty lists all types. + +- ID: `resource_type` +- Datatype: `string` +- Default Value: `None` + + + +### Label selector + +A Hetzner label selector (e.g. env=prod), applied server-side. + +- ID: `label_selector` +- Datatype: `string` +- Default Value: `None` + + + +### Location + +Restrict located resources to a single location name (e.g. fsn1). Empty lists all. Global resources always pass through. + +- ID: `location` +- Datatype: `string` +- Default Value: `None` diff --git a/docs/build/reference/customtask/cmem_plugin_hetzner-ListServerTypes.md b/docs/build/reference/customtask/cmem_plugin_hetzner-ListServerTypes.md new file mode 100644 index 000000000..459fb889e --- /dev/null +++ b/docs/build/reference/customtask/cmem_plugin_hetzner-ListServerTypes.md @@ -0,0 +1,102 @@ +--- +title: "Hetzner Cloud: List Server Types" +description: "List all available Hetzner Cloud server (VM) types, one entity per server type and location, including specs and per-location pricing." +icon: octicons/cross-reference-24 +tags: + - WorkflowTask + - PythonPlugin +--- + +# Hetzner Cloud: List Server Types + + + +!!! note inline end "Python Plugin" + + This operator is part of a Python Plugin Package. + In order to use it, you need to install it, + e.g. with cmemc. + + +This workflow operator lists the available server (virtual machine) types of the +Hetzner Cloud, using the [Hetzner Cloud API](https://docs.hetzner.cloud/). + +It produces one entity per **(server type x location)** offering. Each entity +carries the server type specification (cores, memory, disk, CPU type, +architecture, …), the per-location availability, and the per-location price. + +Booleans are written in their XSD form (`true` / `false`), and a property a +server type does not carry is left out of the entity rather than written as an +empty value. + +## Authentication + +You need a Hetzner Cloud API token of a project (Console → Security → API +Tokens). A read-only token is sufficient. + +## Filters + +All filters are optional. Leave them empty (or `false`) to list everything. A +filter that matches nothing yields an empty result, which the execution report +states. + + +## Parameter + +### API Token + +A Hetzner Cloud project API token (read-only is sufficient). + +- ID: `api_token` +- Datatype: `password` +- Default Value: `None` + +## Advanced Parameter + +### Architecture + +Restrict to a CPU architecture. Empty lists any architecture. + +- ID: `architecture` +- Datatype: `string` +- Default Value: `None` + + + +### CPU Type + +Restrict to a CPU type. Empty lists any CPU type. + +- ID: `cpu_type` +- Datatype: `string` +- Default Value: `None` + + + +### Location + +Restrict to a single location name (e.g. fsn1). Empty lists all. + +- ID: `location` +- Datatype: `string` +- Default Value: `None` + + + +### Only available + +If enabled, list only offerings currently available in their location. + +- ID: `only_available` +- Datatype: `boolean` +- Default Value: `false` + + + +### Exclude deprecated + +If enabled, drop offerings that are deprecated in their location. + +- ID: `exclude_deprecated` +- Datatype: `boolean` +- Default Value: `false` diff --git a/docs/build/reference/customtask/cmem_plugin_jira-JqlQuery.md b/docs/build/reference/customtask/cmem_plugin_jira-JqlQuery.md index 4cdcddc8e..6332b5d68 100644 --- a/docs/build/reference/customtask/cmem_plugin_jira-JqlQuery.md +++ b/docs/build/reference/customtask/cmem_plugin_jira-JqlQuery.md @@ -1,6 +1,6 @@ --- title: "JQL query" -description: "Search and retrieve JIRA issues." +description: "Search and retrieve Jira issues." icon: octicons/cross-reference-24 tags: - WorkflowTask @@ -22,7 +22,7 @@ to the [REST API (v2)](https://developer.atlassian.com/cloud/jira/platform/rest/ Jira service. It is tested both with on-premise Jira deployments as well as with instances on `atlassian.net`. -The result of the JQL query is a list of JIRA issue descriptions (entities). +The result of the JQL query is a list of Jira issue descriptions (entities). This list is forwarded as a JSON document to the output port, where you should connect a JSON Dataset. @@ -84,7 +84,7 @@ Maximum number of issues to retrieve (0 = retrieve all issues). ### Verify SSL Connection - +If enabled, the Jira server's SSL certificate is verified. Disable only when connecting to a server with a self-signed or otherwise untrusted certificate. - ID: `ssl_verify` - Datatype: `boolean` diff --git a/docs/build/reference/customtask/cmem_plugin_kafka-ReceiveMessages.md b/docs/build/reference/customtask/cmem_plugin_kafka-ReceiveMessages.md index f5d8a22d2..5714e6f8a 100644 --- a/docs/build/reference/customtask/cmem_plugin_kafka-ReceiveMessages.md +++ b/docs/build/reference/customtask/cmem_plugin_kafka-ReceiveMessages.md @@ -1,6 +1,6 @@ --- title: "Kafka Consumer (Receive Messages)" -description: "Reads messages from a Kafka topic and saves it to a messages dataset (Consumer)." +description: "Reads messages from a Kafka topic as entities, or saves them to a dataset (Consumer)." icon: octicons/cross-reference-24 tags: - WorkflowTask @@ -19,9 +19,10 @@ tags: This workflow operator uses the Kafka Consumer API -to receive messages from an [Apache Kafka](https://kafka.apache.org/) topic. +to receive messages from an [Apache Kafka](https://kafka.apache.org/) topic. It has +no input port: it starts a workflow by reading from the topic on its own. -Messages received from the topic will be generated as entities with the following +By default, each message received is generated as one entity with the following flat schema: - **key** - the optional key of the message, @@ -36,7 +37,9 @@ In order to process the resulting entities, they have to run through a transform As an alternate working mode, messages can be exported directly to a JSON or XML dataset if you know that the messages on your topic are valid JSON or XML documents -(see Advanced Options > Messages Dataset). +(see the Messages Dataset parameter). In this mode the task has no output port either + +- messages are written directly to the dataset instead of being handed on as entities. In this case, a sample response from the consumer will appear as follows: @@ -239,7 +242,7 @@ The maximum number of messages to fetch and process in each run. If 0 or less, a ### Disable Commit -Setting this to true will disable committing messages after retrival. This means you will receive the same messages on the next execution (for debugging). +Setting this to true will disable committing messages after retrieval. This means you will receive the same messages on the next execution (for debugging). - ID: `disable_commit` - Datatype: `boolean` diff --git a/docs/build/reference/customtask/cmem_plugin_kafka-SendMessages.md b/docs/build/reference/customtask/cmem_plugin_kafka-SendMessages.md index 9efc33e41..21622e50b 100644 --- a/docs/build/reference/customtask/cmem_plugin_kafka-SendMessages.md +++ b/docs/build/reference/customtask/cmem_plugin_kafka-SendMessages.md @@ -1,6 +1,6 @@ --- title: "Kafka Producer (Send Messages)" -description: "Reads a messages dataset and sends records to a Kafka topic (Producer)." +description: "Sends entities, or messages from a dataset, to a Kafka topic (Producer)." icon: octicons/cross-reference-24 tags: - WorkflowTask @@ -18,17 +18,22 @@ tags: e.g. with cmemc. This workflow operator uses the Kafka Producer API to send -messages to a [Apache Kafka](https://kafka.apache.org/). +messages to an [Apache Kafka](https://kafka.apache.org/) topic. -Accepts entities as input, and, if desired, accepts a pre-constructed XML/JSON dataset, -which is transformed into messages and sent to a designated Kafka topic based -on configuration. +By default, each input entity is sent as one message, serialized as JSON with the +entity's URI, schema type and property values; the message key is always empty in +this mode. If a Messages Dataset is configured instead, the input port is replaced +by that dataset, and messages are read from a pre-constructed XML or JSON document +instead of the workflow's entities. This task has no output port: it is a terminal +step and hands nothing on to further operators.
Sample XML format An example XML document is shown below. This document will be sent as two messages - to the configured topic. Each message is created as a proper XML document. + to the configured topic. Each message is created as a proper XML document. A + `` element with more than one child cannot be turned into a single XML + document; it is skipped, and an error is logged. ```xml @@ -113,7 +118,7 @@ on configuration. ### Messages Dataset -Where do you want to retrieve the messages from? The dropdown lists usable datasets from the current project only. In case you miss your dataset, check for the correct type (XML/JSON) and build project). The messages will be retrieved from the entities if no dataset is provided. +Where do you want to retrieve the messages from? The dropdown lists usable datasets from the current project only. In case you miss your dataset, check for the correct type (XML/JSON) and build project. The messages will be retrieved from the entities if no dataset is provided. - ID: `message_dataset` - Datatype: `string` diff --git a/docs/build/reference/customtask/cmem_plugin_ldap-Search.md b/docs/build/reference/customtask/cmem_plugin_ldap-Search.md new file mode 100644 index 000000000..0c5115d4e --- /dev/null +++ b/docs/build/reference/customtask/cmem_plugin_ldap-Search.md @@ -0,0 +1,252 @@ +--- +title: "Search LDAP directory" +description: "Query LDAP directory services such as OpenLDAP or Active Directory." +icon: octicons/cross-reference-24 +tags: + - WorkflowTask + - PythonPlugin +--- + +# Search LDAP directory + + + +!!! note inline end "Python Plugin" + + This operator is part of a Python Plugin Package. + In order to use it, you need to install it, + e.g. with cmemc. + + +Searches a directory server over LDAP and returns each matching entry as an entity. It works with +any LDAPv3 directory, among them OpenLDAP, Active Directory and Apple Open Directory. + +## What the task produces + +The task consumes nothing and produces one entity per directory entry. Each entity carries the +distinguished name of the entry as its first value, followed by the values of the requested +attributes, in the order they were requested. + +A search is normally followed by a transformation which maps the entries into a Knowledge Graph, +with the distinguished name serving as the stable identifier of a person, group or device. + +## How an entry is identified + +The URI of an entity is the LDAP URL of the entry as defined by RFC 4516, built from the URL of +the directory and the percent-encoded distinguished name: + +```text +ldaps://directory.example.org/uid%3Djdoe%2Cou%3Dpeople%2Cdc%3Dexample%2Cdc%3Dorg +``` + +Such a URI stays the same between runs and identifies the entry on its own directory, so two +directories holding the same distinguished name do not collide. The distinguished name is also +available as an ordinary value, which is what a mapping usually builds its own identifier from. + +## How values are returned + +An LDAP attribute may hold several values, and all of them are returned. An attribute an entry +does not carry produces no value at all, which keeps it distinguishable from an attribute that is +present and empty. + +Values arrive in the lexical form a Knowledge Graph expects: timestamps as ISO 8601 dates, +booleans as `true` and `false`, numbers in decimal. A value which is not text - `jpegPhoto`, +`objectSid`, `objectGUID` or `userCertificate`, for instance - is Base64 encoded. + +This works because the schema of the directory is read while connecting, at the cost of one +request. A directory sends every value as text, and only its schema says which of them is a +timestamp, a boolean or a number. + +## Naming the attributes + +The attributes have to be named one by one. `*` and `+` are rejected, because the entities are +described to the workflow while it is being drawn, and a wildcard is only resolved by the +directory once the search runs - there would be nothing to connect the next task to. + +Directories differ in what they call the same thing. A person's login is `uid` on OpenLDAP and +`sAMAccountName` on Active Directory, and group membership is `memberOf` on Active Directory but +often `isMemberOf` elsewhere, so an attribute list is rarely portable between directories. + +## Searching large directories + +Searching is always paged, so a result set is not cut short by the maximum result size of the +directory, which is 1000 entries on a default Active Directory installation. A search which +reaches the configured maximum number of entries stops there and reports a warning saying so, and +a search which exceeds the configured time ends with an error from the directory. + +References to other directory servers are not followed unless this is asked for. Following one +opens a new, anonymous connection to a server named by the directory rather than by this task, +which commonly fails or stalls; Active Directory returns such a reference for every search below +the root of a domain. + +## Connecting securely + +An `ldaps://` URL is encrypted from the start, and an `ldap://` URL can be upgraded before +authenticating where the directory offers no separate encrypted port. Certificates are validated +in both cases. Where the directory uses an internal certificate authority, configure that +authority rather than switching validation off - a connection which accepts any certificate also +accepts one presented by a machine in the middle, and the bind password is sent to it. + +Leaving both the bind name and the password empty binds anonymously, which many directories allow +for a limited part of their data. + +## Example searches + +All people who have a mail address, below one organizational unit: + +```text +(&(objectClass=person)(mail=*)) +``` + +The enabled users of an Active Directory, without the disabled ones: + +```text +(&(objectCategory=person)(objectClass=user)(!(userAccountControl:1.2.840.113556.1.4.803:=2))) +``` + +The members of one group, read from the group entry itself with a base scope: + +```text +(objectClass=groupOfNames) +``` + + +## Parameter + +### LDAP URL + +The URL of the directory server, e.g. `ldaps://directory.example.org:636`. Use `ldaps://` for an encrypted connection (port 636 by default), or `ldap://` for an unencrypted one (port 389 by default), which can be upgraded with the StartTLS parameter. + +- ID: `ldap_url` +- Datatype: `string` +- Default Value: `None` + + + +### Bind DN + +The distinguished name used to authenticate, e.g. `cn=reader,ou=services,dc=example,dc=org`. Leave empty together with the password to bind anonymously. + +- ID: `bind_dn` +- Datatype: `string` +- Default Value: `None` + + + +### Bind password + +The password of the bind DN. Leave empty together with the bind DN to bind anonymously. + +- ID: `bind_password` +- Datatype: `password` +- Default Value: `None` + + + +### Search base + +The distinguished name the search starts at, e.g. `ou=people,dc=example,dc=org`. An empty value is only accepted by directories which allow a search over all naming contexts. + +- ID: `base_dn` +- Datatype: `string` +- Default Value: `None` + + + +### Search filter + +The LDAP filter which selects the entries, in the string representation of RFC 4515, e.g. `(&(objectClass=person)(mail=*))`. + +- ID: `search_filter` +- Datatype: `multiline string` +- Default Value: `(objectClass=*)` + + + +### Scope + +How far below the search base the directory is searched. + +- ID: `scope` +- Datatype: `string` +- Default Value: `SUBTREE` + + + +### Attributes + +The attributes to return, as a comma-separated list, e.g. `cn, mail, memberOf`. Each attribute becomes one value of the produced entities, in the order given here. `*` and `+` are not accepted. + +- ID: `attributes` +- Datatype: `string` +- Default Value: `None` + +## Advanced Parameter + +### Use StartTLS + +If enabled, an `ldap://` connection is upgraded to an encrypted one before authenticating. Enable this where the directory offers encryption on the unencrypted port instead of a separate `ldaps://` endpoint. It cannot be combined with an `ldaps://` URL, which is encrypted already. + +- ID: `start_tls` +- Datatype: `boolean` +- Default Value: `false` + + + +### Verify certificates + +If enabled, the certificate of the directory server is validated against the trusted certificate authorities. Disabling this accepts any certificate, including one presented by a machine-in-the-middle, and sends the bind password to it. Where the directory uses a private certificate authority, provide that authority below instead of disabling validation. + +- ID: `verify_certificates` +- Datatype: `boolean` +- Default Value: `true` + + + +### Certificate authority + +The certificate authority which issued the certificate of the directory server, in PEM format. Needed only when that authority is not among the trusted ones, which is the usual case for a directory with an internal certificate authority. + +- ID: `ca_certificate` +- Datatype: `multiline string` +- Default Value: `None` + + + +### Follow referrals + +If enabled, references to other directory servers are followed. A referral is followed with a new, anonymous connection to a server named by the directory rather than by this task, which commonly fails or stalls. Enable it only for a directory whose data is genuinely partitioned across servers. + +- ID: `follow_referrals` +- Datatype: `boolean` +- Default Value: `false` + + + +### Page size + +How many entries are fetched from the directory per request. This does not limit the number of entries returned overall, only how often the task talks to the directory while collecting them. + +- ID: `page_size` +- Datatype: `Long` +- Default Value: `500` + + + +### Maximum entries + +The maximum number of entries to return, or 0 for no limit. When a search reaches this number, it stops and reports a warning, so that a subtree search over a large directory cannot fill a workflow unnoticed. + +- ID: `size_limit` +- Datatype: `Long` +- Default Value: `0` + + + +### Time limit + +How many seconds the directory may spend on the search, or 0 to leave the decision to the directory. + +- ID: `time_limit` +- Datatype: `Long` +- Default Value: `0` diff --git a/docs/build/reference/customtask/cmem_plugin_llm-CreateEmbeddings.md b/docs/build/reference/customtask/cmem_plugin_llm-CreateEmbeddings.md index f227196b8..dbbc7c479 100644 --- a/docs/build/reference/customtask/cmem_plugin_llm-CreateEmbeddings.md +++ b/docs/build/reference/customtask/cmem_plugin_llm-CreateEmbeddings.md @@ -1,6 +1,6 @@ --- title: "Create Embeddings" -description: "Fetch and output LLM created embeddings from input entities." +description: "Turn the text of each entity into a vector embedding and add it to the entity." icon: octicons/cross-reference-24 tags: - WorkflowTask @@ -18,36 +18,58 @@ tags: e.g. with cmemc. -This plugin creates vector embeddings from text data using an OpenAI compatible embeddings API. -It processes input entities containing text data and generates high-dimensional vector -representations that capture semantic meaning. +Turns the text an entity carries into a vector embedding and adds it to that entity, using an +embeddings model reached over the OpenAI embeddings protocol. Embeddings are what makes +similarity search possible: two entities whose text means roughly the same thing get vectors +that sit close to one another, which is the basis for semantic search, clustering and +retrieval-augmented generation. -## Features +## Ports -- Supports OpenAI embeddings models (e.g., `text-embedding-3-small`) -- Batch processing for efficient API usage -- Configurable input/output paths -- Built-in error handling and workflow cancellation support +Entities arrive on one input port and leave on one output port with two paths added - the +vector, and the text the vector was computed from. Keeping the source text means a later +reader can see exactly what was embedded rather than having to reconstruct it. -## Input/Output +Naming the paths to embed fixes the shape of both ports: the input port asks for those paths +and for any paths you forward, and the output port carries all of them plus the two new ones. +Leaving the selection empty embeds everything an entity carries, which no schema can describe +in advance, so the input port becomes flexible and the output schema is settled only at run +time. -- **Input**: Entities with text data in specified paths -- **Output**: Original entities enhanced with embedding vectors and source text -- Embedding vectors are stored as string representations of float arrays -- Source text used for embedding is preserved for reference +Both added values are text. The vector is the text form of a Python list of floats, such as +`[0.021, -0.118, ...]`, not a JSON array. The source text is the value itself when the entity +holds a single value, or when a single selected path holds a single value; otherwise it is the +text form of a Python dictionary from path to values - again not JSON, so single quotes rather +than double ones. -## Use Cases +## Where it fits + +Embeddings are of no use on their own: they are written on to a store that can search them, +and the usual chain is a dataset or a knowledge graph, then this task, then that store. Its +sibling **Execute Instructions** is the generating half of the same picture, so the two often +appear in one workflow - this task finding the entities worth looking at, and that one writing +about them. + +## Caveats + +Entities are sent in chunks and a whole chunk is embedded in one request. Nothing catches a +failure of that request, so a rate limit, a text longer than the model's context window, or an +entity the provider rejects stops the workflow and takes the rest of the chunk with it. +Cancelling likewise takes effect between chunks. + +An output path may not be a path the incoming entities already carry. This is checked when the +task is configured and again against the schema that actually arrives, and is an error either +way. + +Embeddings are comparable only when they come from the same model. Changing the model means +everything already stored has to be computed again. -- Semantic search and similarity matching -- Text clustering and classification -- Recommendation systems -- Natural language processing pipelines ## Parameter ### Embeddings model -The identifier of the embeddings model to use. Available model IDs for some public providers can be found here: [Claude](https://docs.claude.com/en/docs/build-with-claude/embeddings#available-models), [OpenAI](https://platform.openai.com/docs/guides/embeddings#embedding-models). +The identifier of the embeddings model to use. Some providers do not offer a model listing, in which case type the identifier in directly rather than picking it from the list. Model identifiers for some public providers: [Claude](https://docs.claude.com/en/docs/build-with-claude/embeddings#available-models), [OpenAI](https://platform.openai.com/docs/guides/embeddings#embedding-models). - ID: `model` - Datatype: `string` @@ -57,7 +79,7 @@ The identifier of the embeddings model to use. Available model IDs for some publ ### Embedding entity paths (comma-separated list) -Changing this value will change, which input paths are used by the workflow task to calculate embeddings. A blank value means, all paths are used. +The entity paths whose values are embedded, separated by commas. Naming paths here fixes the schema of both ports. Leaving it empty embeds every path an entity carries and leaves both ports open instead. - ID: `embedding_paths` - Datatype: `string` @@ -67,7 +89,7 @@ Changing this value will change, which input paths are used by the workflow task ### Forward entity paths (comma-separated list) -Paths from input entities to forward to output without modification. These paths will be passed through unchanged alongside embeddings. +Further entity paths carried to the output unchanged, comma-separated. A path named here that an incoming entity does not have arrives empty rather than missing, so the output schema holds whatever was promised. - ID: `forward_paths` - Datatype: `string` @@ -77,7 +99,7 @@ Paths from input entities to forward to output without modification. These paths ### Base URL -The base URL of the OpenAI compatible API (without endpoint path). If left empty CMEMs internal LLM proxy is used. +The base URL of the OpenAI compatible API, without the endpoint path. Left empty, the task uses the built-in large language model proxy of the eccenca Corporate Memory deployment it runs in. That fallback does not apply to `AZURE_OPENAI`, which always needs a URL here. - ID: `base_url` - Datatype: `string` @@ -85,9 +107,9 @@ The base URL of the OpenAI compatible API (without endpoint path). If left empty -### API Type +### API type -Select the API client type. This determines the authentication method and endpoint configuration used for API requests. Choose `OPENAI` for direct OpenAI API access or `AZURE_OPENAI` for Azure-hosted OpenAI services. Consider using the API version advanced parameter in case you access Azure-hosted OpenAI services. +Which client is used to reach the API. `OPENAI` speaks the plain OpenAI protocol and covers OpenAI itself as well as every compatible service. `AZURE_OPENAI` uses Azure's authentication and URL layout, and needs both a base URL and an API version. - ID: `api_type` - Datatype: `enumeration` @@ -97,7 +119,7 @@ Select the API client type. This determines the authentication method and endpoi ### API key -An optional API key for authentication. When using CMEMs internal LLM proxy this parameter is ignored and `EXPLORE_AI_APIKEY` in `environments/config.env` is used. +The key used to authenticate against the API. It is ignored when the built-in model proxy is used, which authenticates with the `EXPLORE_AI_APIKEY` entry of `environments/config.env` instead. - ID: `api_key` - Datatype: `password` @@ -105,9 +127,9 @@ An optional API key for authentication. When using CMEMs internal LLM proxy this -### API Version +### API version -Azure OpenAI API version (only used when API Type is `AZURE_OPENAI`). For more information about OpenAI API version at Azure, please see [the documentation](https://learn.microsoft.com/en-gb/azure/ai-foundry/openai/api-version-lifecycle). +The Azure OpenAI API version, read only when the API type is `AZURE_OPENAI`. The versions currently offered are listed in [the Azure documentation](https://learn.microsoft.com/en-gb/azure/ai-foundry/openai/api-version-lifecycle). - ID: `api_version` - Datatype: `string` @@ -115,9 +137,9 @@ Azure OpenAI API version (only used when API Type is `AZURE_OPENAI`). For more i -### Timeout (milliseconds) +### Timeout (seconds) -The timeout for a single API request in milliseconds. +How long a single API request may take, in seconds. One request covers a whole chunk of entities, so this limits embedding the chunk rather than embedding one entity. - ID: `timout_single_request` - Datatype: `Long` @@ -125,9 +147,9 @@ The timeout for a single API request in milliseconds. -### Entries Processing Buffer +### Entries processing buffer -How many input values do you want to send per request? +How many entities are embedded per API request. Larger chunks mean fewer requests, but a chunk is also the unit that is lost when a request fails, and providers cap how much text one request may carry. - ID: `entries_processing_buffer` - Datatype: `Long` @@ -135,9 +157,9 @@ How many input values do you want to send per request? -### Entity Embedding text (output) +### Entity embedding text (output) -Changing this value will change the output schema accordingly. Default: _embedding_source +The entity path the embedded source text is written to. It may not be a path the incoming entities already carry. - ID: `embedding_output_text` - Datatype: `string` @@ -145,9 +167,9 @@ Changing this value will change the output schema accordingly. Default: _embeddi -### Entity Embedding path (output) +### Entity embedding path (output) -Changing this value will change the output schema accordingly. Default: _embedding +The entity path the vector is written to. It may not be a path the incoming entities already carry. - ID: `embedding_output_path` - Datatype: `string` diff --git a/docs/build/reference/customtask/cmem_plugin_llm-ExecuteInstructions.md b/docs/build/reference/customtask/cmem_plugin_llm-ExecuteInstructions.md index 344c0641d..e04d08aec 100644 --- a/docs/build/reference/customtask/cmem_plugin_llm-ExecuteInstructions.md +++ b/docs/build/reference/customtask/cmem_plugin_llm-ExecuteInstructions.md @@ -1,6 +1,6 @@ --- title: "Execute Instructions" -description: "Send instructions (prompt) to an LLM and process the result." +description: "Prompt a large language model once per entity and add the answer to the entity." icon: octicons/cross-reference-24 tags: - WorkflowTask @@ -18,165 +18,70 @@ tags: e.g. with cmemc. -## Overview +Sends a prompt to a large language model once for every entity it receives, and adds the +model's answer to that entity. The prompt is a Jinja template rendered against the entity, so +every entity gets a prompt of its own. Requests go over the OpenAI chat completions protocol, +which covers OpenAI itself, Azure OpenAI and any service speaking the same protocol - +Anthropic, OpenRouter, or a model hosted locally. With no endpoint configured, the task uses +the built-in model proxy of the eccenca Corporate Memory deployment it runs in. -This plugin executes Large Language Model (LLM) instructions over entity collections, enabling -AI-powered text generation, analysis, and transformation tasks within Corporate Memory workflows. +## Ports -## Core Functionality +Entities arrive on one input port and leave, enriched, on one output port. The shape of both +follows from the instruction prompt alone; variables written into the developer prompt or into +the messages template are not looked at. -- **LLM Integration**: Supports OpenAI API, Azure OpenAI, and OpenAI-compatible endpoints - (Anthropic Claude, OpenRouter, etc.) -- **Entity Processing**: Processes entities individually or in batches with configurable - concurrency -- **Template System**: Uses Jinja2 templates for dynamic prompt generation from entity data -- **File Inputs**: Optionally accept files (images and documents) from CMEM as multimodal - LLM inputs -- **Output Formats**: Supports text, JSON mode, and structured outputs with Pydantic schemas -- **Performance Optimization**: Includes batching, rate limiting, and async processing for - high-throughput scenarios +- **A prompt without variables** is the same for every entity, so there is nothing to read. + The task then has no input port at all and emits a single entity. +- **`{{ entity }}`** renders the whole entity, whose paths cannot be known beforehand. Both + ports are flexible, and everything that arrives is passed through to the output. +- **`{{ entity.title }}`** names one path. The input port then asks for exactly the paths the + prompt names, and the output port carries them through beside the answer. -## Input/Output Behavior +`entity` is the only variable name a prompt may introduce. Any other one is rejected while the +task is being configured rather than at run time. -After processing, each entity receives an additional path (default: `_instruction_output`) -containing the LLM response. For TEXT and JSON_MODE output formats, the response is stored -in this path. For STRUCTURED_OUTPUT, the Pydantic model fields are directly added to the -entity (the `_instruction_output` path is not used). +The answer is added on a path of its own. Under structured output it is the fields of the +schema you supply that become paths instead, and a model nested inside that schema becomes a +sub-entity rather than a path. Switching the task to file input replaces the input side +entirely: the port then takes file entities, one file is sent per request, and there are no +entity paths left for a prompt to read. -Input/output ports are automatically configured based on template variables and the -**Enable File Inputs** option: +## Where it fits -- **No placeholders**: No input ports required -- **With placeholders**: Single input port created for entity data -- **Schema handling**: Fixed schemas when using specific entity paths, flexible schemas otherwise -- **File Inputs enabled**: Single input port accepting `FileEntitySchema` entities (template - variables on the entity are ignored) +The task sits between whatever produces entities - a dataset, a knowledge graph, an earlier +transformation - and whatever consumes them, and leaves what it passes through untouched. +**Create Embeddings** is its counterpart: that task turns text into vectors for retrieval +rather than generating new text, and the two often appear in one workflow, one finding the +entities worth looking at and the other writing about them. -## Template System +## Caveats -Uses Jinja2 templating for dynamic prompts: +The model is asked once per entity, so a run costs one request per entity, and how long it +takes is governed by how many of those may be in flight at a time. -```jinja2 -{{ entity }} # Entire entity as JSON -{{ entity.name }} # Specific entity property -``` - -The following template processing rules are implemented: - -1. **Variable Extraction**: Automatically detects template variables to configure input ports -2. **Entity Iteration**: Processes entities from the single input port individually -3. **Single Entity Context**: Each entity is processed independently with its own template context - -## File Inputs - -Enable **Enable File Inputs** to send files (images or documents) from CMEM to the LLM as -multimodal input. When activated: - -- The input port switches to `FileEntitySchema`, accepting file entities. -- Each file is read from the project, base64-encoded, and embedded into the chat message - as a `data:` URL. -- The `message_template` parameter is ignored and the message structure is selected automatically - by MIME type: - - **Images** (e.g. PNG, JPEG) are sent as `image_url` content parts. - - **Documents** (e.g. PDF, spreadsheets, rich documents, presentations, text and code) are - sent as `file` content parts with the original filename preserved. -- If no MIME type is given, it is guessed by filename extension. -- The `developer_prompt_template` and `instruct_prompt_template` is sent alongside the file. - Entity-derived template variables (e.g. `{{ entity.name }}`) are not available in this mode. -- For **TEXT** and **JSON_MODE** outputs, the output entity additionally carries - `_filename` and `_mimetype` paths so downstream tasks can see what the model saw. - **STRUCTURED_OUTPUT** keeps the schema pure and does not add these paths. - -Note: Multimodal capability depends on the chosen model. Use a vision-capable model for -images and a model that accepts file/document inputs for documents. - -For more details and all available MIME types see the official documentation: - -- [Images and vision](https://developers.openai.com/api/docs/guides/images-vision?api-mode=chat) -- [File inputs](https://developers.openai.com/api/docs/guides/file-inputs?api-mode=chat) - -## Output Formats - -1. **Text Output (Default)** - Standard LLM text responses for general-purpose tasks. -2. **JSON Mode** - Ensures valid JSON output format. Add JSON structure requirements - to your prompt template. -3. **Structured Output** - Uses Pydantic schemas for type-safe, validated responses: - -```python -from pydantic import BaseModel - -class StructuredOutput(BaseModel): - title: str - summary: str - keywords: list[str] - confidence_score: float -``` - -## Performance Features - -Parallel Processing: - -- **Concurrent Requests**: Configurable semaphore-controlled API calls -- **Batch Processing**: Entities processed in configurable batch sizes -- **Rate Limiting**: Optional delays between requests -- **Memory Optimization**: Streaming processing with generator patterns - -Error Handling: - -- **Graceful Degradation**: Continue processing on API errors (configurable) -- **Detailed Logging**: Comprehensive error reporting and debugging information -- **Workflow Integration**: Proper cancellation support and progress reporting - -## API Compatibility - -Supported Providers: - -- **OpenAI**: Direct API access with full feature support -- **Azure OpenAI**: Enterprise Azure-hosted services with API versioning -- **OpenAI-Compatible**: Anthropic Claude, OpenRouter, local models, and other compatible endpoints - -Authentication: - -- **API Keys**: Secure password-type parameters for API authentication -- **Azure Integration**: Supports Azure OpenAI API versioning and endpoint configuration -- **Flexible Endpoints**: Custom base URLs for various providers - -## Advanced Configuration - -### Message Templates - -Customize the conversation structure beyond simple prompts: - -```json -[ - {"role": "system", "content": "You are a data analyst."}, - {"role": "user", "content": "{{ instruction_prompt }}"} -] -``` - -### Performance Tuning - -- **Temperature Control**: Adjust creativity vs. determinism (0.0-2.0) -- **Timeout Management**: Request-level timeout configuration -- **Concurrency Limits**: Prevent rate limiting with request throttling -- **Batch Optimization**: Balance memory usage vs. throughput +Cancelling a workflow takes effect between batches. Requests already sent are finished first. -## Best Practices +Only failures reported by the API count as API errors. A prompt that cannot be rendered, a +messages template that is not valid JSON and a file that cannot be read stop the workflow +whatever the error setting says. -1. **Schema Design**: Use specific entity paths in templates for fixed schemas -2. **Error Strategy**: Enable error continuation for large datasets -3. **Performance**: Adjust concurrency and batch size based on API limits -4. **Templates**: Design prompts with clear instructions and expected outputs -5. **Testing**: Start with small entity sets to validate templates and outputs +Where API errors are not raised, the error text is written to the answer path and the workflow +carries on, so a later task can consume a failure message as though it were an answer. Under +structured output that is worse than it sounds: the declared output schema has no answer path, +the error text is written to one regardless, and the values then no longer line up with the +fields the schema promises. -For detailed prompting guidance, see [OpenAI's Text Generation Guide](https://platform.openai.com/docs/guides/text?api-mode=chat). +JSON mode guarantees only that the answer parses as JSON, never that it has the structure you +had in mind - that still has to be described in the prompt. Structured output does constrain +the shape, but only against providers and models that implement it. ## Parameter -### Instruct Model +### Instruct model -The identifier of the instruct model to use. Note that some provider do not support a model list endpoint. Just create a custom entry then. Available model IDs for some public providers can be found here: [OpenAI](https://platform.openai.com/docs/models), [Claude](https://docs.claude.com/en/docs/about-claude/models/overview), [OpenRouter](https://openrouter.ai/models), [Azure](https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-models/concepts/models-sold-directly-by-azure). **Note:** For STRUCTURED_OUTPUT format, only certain models support structured outputs. See [OpenAI Structured Outputs Guide](https://platform.openai.com/docs/guides/structured-outputs) for supported models. +The identifier of the chat model to ask. Some providers do not offer a model listing, in which case type the identifier in directly rather than picking it from the list. Structured output additionally requires a model that implements it; the provider's own model overview says which ones do. Model identifiers for some public providers: [OpenAI](https://platform.openai.com/docs/models), [Claude](https://docs.claude.com/en/docs/about-claude/models/overview), [OpenRouter](https://openrouter.ai/models), [Azure](https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-models/concepts/models-sold-directly-by-azure). - ID: `model` - Datatype: `string` @@ -184,9 +89,9 @@ The identifier of the instruct model to use. Note that some provider do not supp -### Developer Prompt Template +### Developer prompt template -The developer (system) prompt inserted at `{{ developer_prompt }}` in the Messages Template. Defines the LLM's role and behaviour. +The developer (system) prompt, which sets the model's role and behaviour. It is substituted into the messages template wherever `{{ developer_prompt }}` appears, but it is not itself rendered against the entity: Jinja variables written here stay in the prompt as literal text. - ID: `developer_prompt_template` - Datatype: `code-jinja2` @@ -194,9 +99,9 @@ The developer (system) prompt inserted at `{{ developer_prompt }}` in the Messag -### Instruction Prompt Template +### Instruction prompt template -The instruction (user) prompt inserted at `{{ instruction_prompt }}` in the Messages Template. Please have a look at the task documentation for detailed instructions. +The instruction (user) prompt, the actual request made of the model. It is rendered against each entity and the result is substituted into the messages template wherever `{{ instruction_prompt }}` appears. The variables it uses decide whether the task has an input port and what that port asks for, which the task documentation describes in full. - ID: `instruct_prompt_template` - Datatype: `code-jinja2` @@ -204,9 +109,9 @@ The instruction (user) prompt inserted at `{{ instruction_prompt }}` in the Mess -### Enable File Inputs +### Enable file inputs -Send files from CMEM to the LLM as multimodal input. When enabled, the input port accepts files (FileEntitySchema). Each file is base64-encoded and embedded into the chat message: images (e.g. PNG, JPEG) are sent as image content parts; documents and other files (e.g. PDF, spreadsheets, rich documents, presentations, text and code) are sent as file content parts. Which MIME types are supported is highly dependent on the provider and model used. Entity-derived template variables (e.g. `{{ entity.name }}`) are not available in this mode. The `developer_prompt_template` as well as `instruct_prompt_template` are sent alongside the image/file. For TEXT and JSON_MODE outputs, the output entity carries `_filename` and `_mimetype` paths so downstream tasks can see what the model saw. Note: requires a multimodal-capable model for the chosen file type. +If enabled, the task sends files instead of entity data. The input port then accepts file entities, and each file is base64 encoded into its own request: images go as image content parts and everything else - PDFs, spreadsheets, presentations, text and code - as file content parts, with the filename preserved. A file carrying no MIME type is classified by its extension. The model has to be able to read the type you send it, and which types a provider accepts varies widely. Both prompts are sent alongside the file, but there is no entity left for them to read, so entity variables in the instruction prompt have nothing to render. The messages template is replaced by a built-in one. For text and JSON mode the output additionally carries `_filename` and `_mimetype`, so a later task can see what the model saw; structured output keeps its schema to the declared fields and adds neither. - ID: `enable_file_inputs` - Datatype: `boolean` @@ -216,7 +121,7 @@ Send files from CMEM to the LLM as multimodal input. When enabled, the input por ### Base URL -The base URL of the OpenAI compatible API (without endpoint path). If left empty CMEMs internal LLM proxy is used. +The base URL of the OpenAI compatible API, without the endpoint path. Left empty, the task uses the built-in large language model proxy of the eccenca Corporate Memory deployment it runs in. That fallback does not apply to `AZURE_OPENAI`, which always needs a URL here. - ID: `base_url` - Datatype: `string` @@ -224,9 +129,9 @@ The base URL of the OpenAI compatible API (without endpoint path). If left empty -### API Type +### API type -Select the API client type. This determines the authentication method and endpoint configuration used for API requests. Choose `OPENAI` for direct OpenAI API access or `AZURE_OPENAI` for Azure-hosted OpenAI services. Consider using the API version advanced parameter in case you access Azure-hosted OpenAI services. +Which client is used to reach the API. `OPENAI` speaks the plain OpenAI protocol and covers OpenAI itself as well as every compatible service. `AZURE_OPENAI` uses Azure's authentication and URL layout, and needs both a base URL and an API version. - ID: `api_type` - Datatype: `enumeration` @@ -236,7 +141,7 @@ Select the API client type. This determines the authentication method and endpoi ### API key -An optional API key for authentication. When using CMEMs internal LLM proxy this parameter is ignored and `EXPLORE_AI_APIKEY` in `environments/config.env` is used. +The key used to authenticate against the API. It is ignored when the built-in model proxy is used, which authenticates with the `EXPLORE_AI_APIKEY` entry of `environments/config.env` instead. - ID: `api_key` - Datatype: `password` @@ -244,9 +149,9 @@ An optional API key for authentication. When using CMEMs internal LLM proxy this -### API Version +### API version -Azure OpenAI API version (only used when API Type is `AZURE_OPENAI`). For more information about OpenAI API version at Azure, please see [the documentation](https://learn.microsoft.com/en-gb/azure/ai-foundry/openai/api-version-lifecycle). +The Azure OpenAI API version, read only when the API type is `AZURE_OPENAI`. The versions currently offered are listed in [the Azure documentation](https://learn.microsoft.com/en-gb/azure/ai-foundry/openai/api-version-lifecycle). - ID: `api_version` - Datatype: `string` @@ -254,9 +159,9 @@ Azure OpenAI API version (only used when API Type is `AZURE_OPENAI`). For more i -### Temperature (between 0 and 2) +### Temperature -A parameter that controls the randomness and creativity of the model. A high temperature value (`0.8` - `1.0`) increases randomness and creativity. This is useful for open-ended tasks like storytelling or brainstorming. A low temperature value (`0.0` - `0.4`) produces more deterministic and focused outputs. This is suitable for factual or technical tasks. +How random the model is allowed to be, between `0` and `2`. A high value (`0.8` - `1.0`) makes the output more varied and more creative, which suits open-ended writing. A low value (`0.0` - `0.4`) makes it more repeatable and more focused, which suits factual and technical work. - ID: `temperature` - Datatype: `double` @@ -266,7 +171,7 @@ A parameter that controls the randomness and creativity of the model. A high tem ### Timeout (seconds) -The timeout for a single API request in seconds. +How long a single API request may take before it is abandoned, in seconds. - ID: `timeout` - Datatype: `double` @@ -274,9 +179,9 @@ The timeout for a single API request in seconds. -### Instruction Output Path +### Instruction output path -The entity path where the instruction result will be provided. Note: This parameter is not used when Output Format is set to STRUCTURED_OUTPUT. For structured outputs, only the Pydantic model fields are included in the output schema. +The entity path the model's answer is written to. It may not be a path the prompt reads. Structured output does not use it, because the fields of the schema become the output paths instead. - ID: `instruction_output_path` - Datatype: `string` @@ -284,9 +189,9 @@ The entity path where the instruction result will be provided. Note: This parame -### Messages Template +### Messages template -A list of messages comprising the conversation compatible with OpenAI chat completion API message object. Ignored if `enable_file_inputs` is active. Have look at [Message roles and instruction following](https://platform.openai.com/docs/guides/text#message-roles-and-instruction-following) to learn about different levels of priority to messages with different roles. +The list of messages sent to the chat completions API, as JSON. `{{ developer_prompt }}` and `{{ instruction_prompt }}` are replaced by the two prompts, and the template has to mention `instruction_prompt` somewhere. Use it to add further messages, to give an example exchange, or to assign the prompts to roles other than the default ones - [message roles and instruction following](https://platform.openai.com/docs/guides/text#message-roles-and-instruction-following) explains what the roles weigh. It is replaced by a built-in template when file inputs are enabled. - ID: `messages_template` - Datatype: `code-json` @@ -307,9 +212,9 @@ A list of messages comprising the conversation compatible with OpenAI chat compl -### Output Format +### Output format -Specifying the format that the model must output. Possible values are `TEXT` - Standard text output, `STRUCTURED_OUTPUT` - output follows a given schema. Add your schema as Pydantic model in the parameter below, `JSON_MODE` - a more basic version of the structured outputs feature where you have to add your structure to the prompt template. +What the model is asked to return. `TEXT` takes the answer as it comes. `JSON_MODE` makes the provider guarantee that the answer parses as JSON, while the structure it should have remains something to describe in the prompt. `STRUCTURED_OUTPUT` hands the provider a schema instead, so the answer comes back in the shape the Pydantic schema gives. - ID: `output_format` - Datatype: `enumeration` @@ -317,9 +222,9 @@ Specifying the format that the model must output. Possible values are `TEXT` - S -### Pydantic Schema +### Pydantic schema -The Pydantic schema definition with a mandatory class named `StructuredOutput(BaseModel)`. This is only used in combination with the Structured Output format. A schema may have up to 100 object properties total, with up to 5 levels of nesting. The total string length of all property names, definition names, enum values, and const values cannot exceed 15,000 characters. +The schema the answer has to follow, used by the structured output format. It is a Pydantic model, and the root class has to be named `StructuredOutput`. Only class definitions with typed fields are accepted here, next to imports from `pydantic`, `typing`, `typing_extensions`, `datetime` and `enum`. Methods, plain assignments and anything else executable are rejected. Nested models are allowed and become sub-entities on the output. Providers place their own limits on the schema, commonly at 100 properties, five levels of nesting, and 15,000 characters across all property, definition, enum and constant names. - ID: `pydantic_schema` - Datatype: `code-python` @@ -339,7 +244,7 @@ class StructuredOutput(BaseModel): ### Raise on API errors -How to react on API errors. When enable, any API errors will cause the workflow to stop with an exception. When disabled, API errors are logged and the error message is written to the entity output, allowing the workflow to continue processing other entities. +If enabled, an error reported by the API stops the workflow. If disabled, the error is logged, its text is written to the output in place of an answer, and the remaining entities are still processed. Errors that are not the API's - an unrenderable prompt, an unreadable file - stop the workflow either way. - ID: `raise_on_error` - Datatype: `boolean` @@ -347,9 +252,9 @@ How to react on API errors. When enable, any API errors will cause the workflow -### Maximum Concurrent Requests +### Maximum concurrent requests -Maximum number of concurrent API requests to prevent rate limiting and resource exhaustion. +How many API requests may be in flight at once. This is what governs how fast a run gets through its entities, and lowering it is the first answer to a provider that rate-limits. - ID: `max_concurrent_requests` - Datatype: `Long` @@ -357,9 +262,9 @@ Maximum number of concurrent API requests to prevent rate limiting and resource -### Batch Size +### Batch size -Number of entities to process in each batch for memory optimization. +How many entities are prepared and processed together. Entities are read and handed on one batch at a time rather than all at once, so a smaller batch keeps less in memory and lets results reach the next task sooner. - ID: `batch_size` - Datatype: `Long` @@ -367,9 +272,9 @@ Number of entities to process in each batch for memory optimization. -### Request Delay (seconds) +### Request delay (seconds) -Delay between API requests in seconds to respect rate limits. +How long to wait before each API request, in seconds. Left at zero, requests are sent as fast as the concurrency limit allows. Raise it for a provider that rejects bursts even within its request limit. - ID: `request_delay` - Datatype: `double` diff --git a/docs/build/reference/customtask/cmem_plugin_mattermost.md b/docs/build/reference/customtask/cmem_plugin_mattermost.md index ab78b6541..c2c464ee5 100644 --- a/docs/build/reference/customtask/cmem_plugin_mattermost.md +++ b/docs/build/reference/customtask/cmem_plugin_mattermost.md @@ -1,6 +1,6 @@ --- title: "Send Mattermost messages" -description: "Send messages to Mattermost channels and/or users." +description: "Post messages to Mattermost users and channels through a bot account." icon: octicons/cross-reference-24 tags: - WorkflowTask @@ -17,31 +17,43 @@ tags: In order to use it, you need to install it, e.g. with cmemc. -This task sends messages to Mattermost channels and users. - -You need a bot account in order to connect to Mattermost. -Learn more on bot accounts at +This task posts messages to Mattermost. Every message is sent by a bot +account, so recipients see it as coming from the bot rather than from the +person who ran the workflow. You need that bot account and a personal access +token for it before the task can do anything - learn more at [developers.mattermost.com](https://developers.mattermost.com): - [Using bot accounts](https://developers.mattermost.com/integrate/reference/bot-accounts/) - [Personal access tokens](https://developers.mattermost.com/integrate/reference/personal-access-token/) -The task has two working modes. +# Ports -# Single Message +One input port accepts entities carrying the paths `user`, `channel` and +`message`. Each entity is sent as one message per recipient it names: a direct +message to the user, a post to the channel, or both. -You can send a single static message to a pre-configured channel or user. -Just configure the User and/or Channel and Message parameters to do so. +There is no output port. The task is a terminal step - it hands nothing on to +the rest of the workflow and reports only how many messages went where. -# Multiple Messages +A recipient and a message configured on the task itself are sent once, before +any entity is read. Configuring them while entities also arrive sends both: +that message once, and one message per entity on top of it. -You can send multiple messages to different channels or users by piping data into -the task. For each entity, a message is send. For dynamic messages the following -input paths are recognized: +# Caveats -- user -- channel -- message +- One field names one recipient, matched exactly. A user is found by username, + nickname, email address or full name; a channel by its name or its display + name. A comma separated list is **not** split into several recipients - it is + looked up as a single name, which fails. +- What an entity carries is not merged with what is configured on the task. An + entity without a message does not fall back to the configured message; it + fails. +- The task stops at the first entity it cannot send - one naming no recipient, + one carrying no message, or one naming a recipient Mattermost does not know. + Messages sent for earlier entities have already been posted and are not taken + back. +- Requests to Mattermost time out after two seconds, so an unreachable or slow + server fails the task rather than delaying it. ## Parameter @@ -68,7 +80,7 @@ The Personal Access Token of the bot account. ### Bot name -The name or display name of the bot you want to use to connect. +The bot account that sends the messages, named by its username, nickname, email address or full name. - ID: `bot_name` - Datatype: `string` @@ -78,7 +90,7 @@ The name or display name of the bot you want to use to connect. ### User -The user account which will receive the message. You can search for users if the connection was successful (Base URl, bot + token). +The single user account that receives the message as a direct message, named by username, nickname, email address or full name. Type at least one letter to search for users. Suggestions need the URL and the Access Token to be filled in first. - ID: `user` - Datatype: `string` @@ -88,7 +100,7 @@ The user account which will receive the message. You can search for users if the ### Channel -The channel which will receive the message. You can search for channels if the connection was successful (Base URl, bot + token). If you want to send your message to multiple channels, separate them with a comma. +The single channel that receives the message, named by its name or its display name. Type at least one letter to search for channels. Suggestions need the URL and the Access Token to be filled in first. - ID: `channel` - Datatype: `string` @@ -98,7 +110,7 @@ The channel which will receive the message. You can search for channels if the c ### Message -The message size is limited to a configured maximum (e.g. 16383 characters). +The message text, interpreted by Mattermost as Markdown. Its maximum length is a server setting, 16383 characters by default. - ID: `message` - Datatype: `multiline string` diff --git a/docs/build/reference/customtask/cmem_plugin_ntfy-GetAttachments.md b/docs/build/reference/customtask/cmem_plugin_ntfy-GetAttachments.md new file mode 100644 index 000000000..667607b73 --- /dev/null +++ b/docs/build/reference/customtask/cmem_plugin_ntfy-GetAttachments.md @@ -0,0 +1,178 @@ +--- +title: "Get ntfy attachments" +description: "Download the attachments of a topic's messages from an ntfy server." +icon: octicons/cross-reference-24 +tags: + - WorkflowTask + - PythonPlugin +--- + +# Get ntfy attachments + + + +!!! note inline end "Python Plugin" + + This operator is part of a Python Plugin Package. + In order to use it, you need to install it, + e.g. with cmemc. + +## What it does + +Downloads the files attached to the messages of a topic, so a workflow can +process what somebody sent to it. + +## What flows through the ports + +The task consumes nothing and reads from the server itself. Its output port +carries one file entity per downloaded attachment, with the media type the +message declared and the name it gave, reduced to a plain file name. A file +dataset or a file consuming task can be connected to it. + +## Where it sits + +It reads the same topic as **Get ntfy messages** and keeps its own position, so +the two can run side by side without taking each other's messages. They share +one position file by default, which is worth separating if they are scheduled +to finish at the same moment. Use that task instead when the text of the +messages is what matters. + +## Caveats + +- **A message without an attachment is passed over**, and so is an attachment + that is not where the message said - ntfy drops an uploaded file after three + hours. Neither fails the run, and the report counts them apart, because the + second one is also what a wrong address looks like from here. +- **A run either reads everything or only what is new**, as with the messages + task, and the position advances over passed-over messages as well, so they + are not offered again. +- **A message published a moment ago may not be there yet.** A busy server + writes its message cache in batches - ntfy.sh is configured for one second - + so a workflow that publishes to a topic and reads it back in the same run can + come away with nothing. Reading what another run published is reliable; + reading what this run published is not. +- **An attachment is fetched from wherever the message points.** A publisher + may attach a file that is hosted elsewhere, and ntfy hands such a URL on + unchanged, so the download goes to that host rather than to the configured + server. The access token is sent to the configured server only. +- **ntfy writes those addresses from its own base URL**, not from the address + it was reached on, so a server behind a container host name, a proxy or a + tunnel hands out links that point somewhere this task cannot reach - and that + it does not recognise as the configured server either, which withholds the + token as well. The task can be told to fetch ntfy's own file path from the + configured server instead, which leaves an attachment hosted elsewhere alone. +- **Files are downloaded to the worker's temporary directory** and are not + cleaned up by the task, because the rest of the workflow still has to read + them. +- **An attachment may be large, and only this task limits it.** ntfy allows + 15 MB per file by default, but an attachment hosted elsewhere is not bound by + that, so anything over the size limit configured here is passed over and + counted rather than written to the worker's disk. +- **An unprotected topic is public, and anybody may write to it.** A topic + needs no sign-up and carries no password of its own, so on a server without + access control anybody who guesses the name can publish to it - which, + together with the point above, means choosing what this task downloads and + from where. Give the topic a long random suffix rather than a word like + `reports`, and protect it with the server's own + [access control](https://docs.ntfy.sh/config/#access-control) where that is available. + + +## Parameter + +### Server + +The base URL of the ntfy server. The default is the public service; a self-hosted ntfy is named here instead, for example - see [installing ntfy](https://docs.ntfy.sh/install/). + +- ID: `server` +- Datatype: `string` +- Default Value: `https://ntfy.sh` + + + +### Access token + +The access token to authenticate with. A public topic needs none, so this can be left empty. See [access tokens](https://docs.ntfy.sh/config/#access-tokens). + +- ID: `access_token` +- Datatype: `password` +- Default Value: `None` + + + +### Topic + +The topic whose attachments are downloaded. A topic needs no sign-up and has no password of its own, so give it a long random suffix nobody can guess rather than a plain word - see [picking a topic](https://docs.ntfy.sh/publish/#picking-a-topic). + +- ID: `topic` +- Datatype: `string` +- Default Value: `None` + + + +### Messages to return + +Which of a topic's messages a run returns. + +- ID: `read_mode` +- Datatype: `string` +- Default Value: `stateless` + + + +### Starting point + +How far back a run reaches: a duration such as `10m`, a Unix timestamp, a message id, or `all` for everything the server still holds. A task set to return only new messages reads this on its first run only, and continues from its remembered position afterwards. See [fetching cached messages](https://docs.ntfy.sh/subscribe/api/#fetch-cached-messages). + +- ID: `since` +- Datatype: `string` +- Default Value: `all` + +## Advanced Parameter + +### Priority filter + +Return only messages of these priorities, as a comma separated list - each one a number from 1 to 5 or one of ntfy's names, from `min` to `max`. A message matching any of them is returned. Empty returns every priority. See [filtering messages](https://docs.ntfy.sh/subscribe/api/#filter-messages). + +- ID: `priority_filter` +- Datatype: `string` +- Default Value: `None` + + + +### Tag filter + +Return only messages carrying these tags, as a comma separated list. A message must carry all of them. Empty returns every message. See [filtering messages](https://docs.ntfy.sh/subscribe/api/#filter-messages). + +- ID: `tags_filter` +- Datatype: `string` +- Default Value: `None` + + + +### Maximum attachment size + +How many megabytes an attachment may weigh before it is passed over rather than downloaded. An attachment URL is whatever the publisher of the message put there, and ntfy forwards an external one unchanged, so the server's own limit of 15 MB does not bind it. 0 downloads whatever arrives. + +- ID: `maximum_size` +- Datatype: `Long` +- Default Value: `100` + + + +### Fetch attachments from this server + +If enabled, an attachment is fetched from the server configured above rather than from the address the message carries. ntfy builds that address from its own base URL, so a server reached under another name - a container host name, a proxy, a tunnel - hands out links that point somewhere unreachable and that the task does not recognise as its own, which also withholds the access token. Only ntfy's own file path is redirected; an attachment the publisher hosted elsewhere is always fetched from where it says. + +- ID: `from_own_server` +- Datatype: `boolean` +- Default Value: `false` + + + +### Position file + +The project file the read positions are kept in. One file holds the positions of every task in the project and is written whole, so give a task its own file when it may finish at the same moment as another one reading the same project. + +- ID: `cursor_file` +- Datatype: `string` +- Default Value: `ntfy-cursors.json` diff --git a/docs/build/reference/customtask/cmem_plugin_ntfy-GetMessages.md b/docs/build/reference/customtask/cmem_plugin_ntfy-GetMessages.md new file mode 100644 index 000000000..d8188710a --- /dev/null +++ b/docs/build/reference/customtask/cmem_plugin_ntfy-GetMessages.md @@ -0,0 +1,155 @@ +--- +title: "Get ntfy messages" +description: "Read the messages of a topic from an ntfy server." +icon: octicons/cross-reference-24 +tags: + - WorkflowTask + - PythonPlugin +--- + +# Get ntfy messages + + + +!!! note inline end "Python Plugin" + + This operator is part of a Python Plugin Package. + In order to use it, you need to install it, + e.g. with cmemc. + +## What it does + +Reads the messages a topic holds and hands them on as entities, so a workflow +can act on what arrived through ntfy. + +## What flows through the ports + +The task consumes nothing and reads from the server itself. Its output port +carries one entity per message, with the paths `id`, `time`, `time_iso`, +`topic`, `title`, `message`, `priority`, `tags`, `click`, `icon`, the four +`attachment_*` paths, and `json` holding the message verbatim - so a field this +task does not model can still be read downstream. `tags` is the only path that +may carry several values; every other path carries one or none. A message that +was sent without a priority carries 3, the default ntfy applies to it and +leaves out of its answer. + +## Where it sits + +It is the reading half of this package. **Get ntfy attachments** fetches the +files of those messages, and **Send ntfy message** sends in the other +direction. + +## Caveats + +- **A run either returns everything or only what is new.** Returning only new + ones makes the task write the position it reached into a file in the project + and continue from it next time. Each task has its own entry there, so two + tasks reading one topic never take each other's messages - but the file is + written whole, so two of them finishing in the same moment can lose one of + the two positions, and that task reads its window again. +- **The position only advances over messages a consumer asked past.** Taking + an entity is not proof that it was processed, so the position follows the + last message the rest of the workflow came back from. A run failing halfway + leaves everything after that to be read again: a message is delivered at + least once, duplicates are possible, and a lost message is not. A worker + killed outright writes no position at all. +- **A message published a moment ago may not be there yet.** A busy server + writes its message cache in batches - ntfy.sh is configured for one second - + so a workflow that publishes to a topic and reads it back in the same run can + come away with nothing. Reading what another run published is reliable; + reading what this run published is not. +- **A message id that the server has forgotten cannot be resumed from.** ntfy + keeps messages for twelve hours by default, so a task that has not run for + longer starts from the whole window again - duplicates rather than loss. +- **The preview shows what the starting point names**, not the position a run + would continue from, because a task action is not told which task it belongs + to. Resetting the position has the same limitation and clears the position of + every task reading this topic in this project. +- **An unprotected topic is public, and anybody may write to it.** A topic + needs no sign-up and carries no password of its own, so on a server without + access control anybody who guesses the name can read what is on it and + publish to it - and this task hands a stranger's message to the workflow as + readily as its own. Give the topic a long random suffix rather than a word + like `alerts`, and protect it with the server's own + [access control](https://docs.ntfy.sh/config/#access-control) where that is available. + + +## Parameter + +### Server + +The base URL of the ntfy server. The default is the public service; a self-hosted ntfy is named here instead, for example - see [installing ntfy](https://docs.ntfy.sh/install/). + +- ID: `server` +- Datatype: `string` +- Default Value: `https://ntfy.sh` + + + +### Access token + +The access token to authenticate with. A public topic needs none, so this can be left empty. See [access tokens](https://docs.ntfy.sh/config/#access-tokens). + +- ID: `access_token` +- Datatype: `password` +- Default Value: `None` + + + +### Topic + +The topic whose messages are read. A topic needs no sign-up and has no password of its own, so give it a long random suffix nobody can guess rather than a plain word - see [picking a topic](https://docs.ntfy.sh/publish/#picking-a-topic). + +- ID: `topic` +- Datatype: `string` +- Default Value: `None` + + + +### Messages to return + +Which of a topic's messages a run returns. + +- ID: `read_mode` +- Datatype: `string` +- Default Value: `stateless` + + + +### Starting point + +How far back a run reaches: a duration such as `10m`, a Unix timestamp, a message id, or `all` for everything the server still holds. A task set to return only new messages reads this on its first run only, and continues from its remembered position afterwards. See [fetching cached messages](https://docs.ntfy.sh/subscribe/api/#fetch-cached-messages). + +- ID: `since` +- Datatype: `string` +- Default Value: `all` + +## Advanced Parameter + +### Priority filter + +Return only messages of these priorities, as a comma separated list - each one a number from 1 to 5 or one of ntfy's names, from `min` to `max`. A message matching any of them is returned. Empty returns every priority. See [filtering messages](https://docs.ntfy.sh/subscribe/api/#filter-messages). + +- ID: `priority_filter` +- Datatype: `string` +- Default Value: `None` + + + +### Tag filter + +Return only messages carrying these tags, as a comma separated list. A message must carry all of them. Empty returns every message. See [filtering messages](https://docs.ntfy.sh/subscribe/api/#filter-messages). + +- ID: `tags_filter` +- Datatype: `string` +- Default Value: `None` + + + +### Position file + +The project file the read positions are kept in. One file holds the positions of every task in the project and is written whole, so give a task its own file when it may finish at the same moment as another one reading the same project. + +- ID: `cursor_file` +- Datatype: `string` +- Default Value: `ntfy-cursors.json` diff --git a/docs/build/reference/customtask/cmem_plugin_ntfy-SendAttachment.md b/docs/build/reference/customtask/cmem_plugin_ntfy-SendAttachment.md new file mode 100644 index 000000000..68d022a0d --- /dev/null +++ b/docs/build/reference/customtask/cmem_plugin_ntfy-SendAttachment.md @@ -0,0 +1,143 @@ +--- +title: "Send ntfy attachment" +description: "Send a file as an attachment to a topic on an ntfy server." +icon: octicons/cross-reference-24 +tags: + - WorkflowTask + - PythonPlugin +--- + +# Send ntfy attachment + + + +!!! note inline end "Python Plugin" + + This operator is part of a Python Plugin Package. + In order to use it, you need to install it, + e.g. with cmemc. + +## What it does + +Sends files to an ntfy topic as attachments, so a report a workflow just +produced arrives where somebody will see it. + +## What flows through the ports + +One input port accepts file entities - from a file dataset, a project resource +or a preceding task that produces files - and every file that arrives is sent +as one notification, byte for byte, under its own name. There is no output +port: the task is a terminal step and hands nothing on. + +## Where it sits + +It is the file sending counterpart of **Send ntfy message**, which sends text, +and of **Get ntfy attachments**, which fetches files in the other direction. + +## Caveats + +- **The file is the body of the notification**, so the text fields configured + here travel beside it and describe it rather than replacing it. +- **An input is required.** With nothing connected the task has no file to + send and fails instead of notifying anybody. +- **ntfy limits an attachment to 15 MB** by default and keeps it for three + hours, after which the recipient's link stops working. A file the server + refuses fails the run, and files already sent are not taken back. +- **The media type a recipient sees comes from the file name.** ntfy reads it + from the name's extension and ignores what the request declares, so a file + entity's own media type does not travel with it and a file arriving without + an extension is offered as a generic download. +- **An upload is never retried, not even when the server asks for a slower + pace.** A file arrives as a stream that can be read once, so a second attempt + would upload an empty attachment that ntfy accepts and stores. Failing on the + rate limit and re-running the workflow is the honest outcome. +- **What is configured applies to every file.** A per-file title or priority + is not read from the entities, which carry a file and nothing else. +- **An unprotected topic is public.** A topic needs no sign-up and carries no + password of its own, so on a server without access control anybody who + guesses the name can download every file this task sends - and a file is + usually a whole report rather than one line of text. Give the topic a long + random suffix rather than a word like `reports`, and protect it with the + server's own [access control](https://docs.ntfy.sh/config/#access-control) + where that is available. + + +## Parameter + +### Server + +The base URL of the ntfy server. The default is the public service; a self-hosted ntfy is named here instead, for example - see [installing ntfy](https://docs.ntfy.sh/install/). + +- ID: `server` +- Datatype: `string` +- Default Value: `https://ntfy.sh` + + + +### Access token + +The access token to authenticate with. A public topic needs none, so this can be left empty. See [access tokens](https://docs.ntfy.sh/config/#access-tokens). + +- ID: `access_token` +- Datatype: `password` +- Default Value: `None` + + + +### Topic + +The topic the files are sent to. A topic needs no sign-up and has no password of its own, so give it a long random suffix nobody can guess rather than a plain word - see [picking a topic](https://docs.ntfy.sh/publish/#picking-a-topic). + +- ID: `topic` +- Datatype: `string` +- Default Value: `None` + + + +### Title + +The title shown with every attachment of this task. It travels as a header, so it is a single line and a line break becomes a space. See [message titles](https://docs.ntfy.sh/publish/#message-title). + +- ID: `title` +- Datatype: `string` +- Default Value: `None` + + + +### Message + +The text shown with every attachment of this task. Left empty, the recipient sees the file name. Like the title it is a single line, and anything beyond 4096 bytes is shortened. + +- ID: `message` +- Datatype: `string` +- Default Value: `None` + + + +### Priority + +How urgently every notification of this task is shown. See [message priority](https://docs.ntfy.sh/publish/#message-priority). + +- ID: `priority` +- Datatype: `string` +- Default Value: `default` + + + +### Notification tags + +Tags shown with the notification, as a comma separated list. A tag that names an emoji is shown as that emoji. See [tags and emojis](https://docs.ntfy.sh/publish/#tags-emojis) and the [list of emoji names](https://docs.ntfy.sh/emojis/). + +- ID: `notification_tags` +- Datatype: `string` +- Default Value: `None` + +## Advanced Parameter + +### Click URL + +The URL opened when the notification is tapped. See [click action](https://docs.ntfy.sh/publish/#click-action). + +- ID: `click` +- Datatype: `string` +- Default Value: `None` diff --git a/docs/build/reference/customtask/cmem_plugin_ntfy-SendMessage.md b/docs/build/reference/customtask/cmem_plugin_ntfy-SendMessage.md new file mode 100644 index 000000000..7383c127d --- /dev/null +++ b/docs/build/reference/customtask/cmem_plugin_ntfy-SendMessage.md @@ -0,0 +1,192 @@ +--- +title: "Send ntfy message" +description: "Send a push notification to a topic on an ntfy server." +icon: octicons/cross-reference-24 +tags: + - WorkflowTask + - PythonPlugin +--- + +# Send ntfy message + + + +!!! note inline end "Python Plugin" + + This operator is part of a Python Plugin Package. + In order to use it, you need to install it, + e.g. with cmemc. + +## What it does + +Sends a push notification through ntfy, so a workflow can tell somebody what +it did without them watching the workspace. + +## What flows through the ports + +Notifications arrive on one input port as entities carrying the paths `topic`, +`title`, `message`, `priority`, `tags` and `click`, and one notification is +sent per entity - where an entity's `priority` is a number from 1 to 5, one of +ntfy's names from `min` to `max`, or `urgent`, which ntfy reads as `max`. That +port is replaced by the task's own fields when a message is configured on the +task itself, which then sends exactly one notification. There is no output port +either way: the task is a terminal step and hands nothing on. + +## Where it sits + +It is the sending half of this package. **Get ntfy messages** reads a topic in +the other direction, and **Send ntfy attachment** sends a file produced earlier +in the workflow. + +## Caveats + +- **The two ways of supplying a notification exclude each other.** Either the + task sends the one notification configured on it, or it sends one per entity + that arrives - there is no run in which a constant message goes out a hundred + times. Markdown, Delay, Icon URL and Email copy are the exception: they are + never carried by an entity and apply either way. +- **The topic is the one field an entity may leave to the task.** An entity + that names no topic is sent to the topic configured here, which spares a + workflow from attaching a constant to every entity. +- **An entity that carries no message fails the run**, because a notification + without text says nothing. So does a run that has neither: a task with no + message configured and nothing connected to its port fails rather than + sending silence. +- **The first rejected request fails the run.** A rate limit is retried up to + three times first, since the server has said it took nothing. A server error + is not retried at all: it may have delivered the notification before failing + to answer, and a duplicate alert is worse than a failed run. Notifications + already sent are not taken back, and the report says how many went out + before the failure. +- **A message longer than 4096 bytes is shortened**, marked with an ellipsis, + rather than refused by the server. The report counts how often that happened. +- **An unprotected topic is public.** A topic needs no sign-up and carries no + password of its own, so on a server without access control the only thing + between a notification and anybody who guesses the name is the name itself - + and then every title and message a workflow sends is readable, along with + what ran and when. Give the topic a long random suffix rather than a word + like `alerts`, and protect it with the server's own + [access control](https://docs.ntfy.sh/config/#access-control) where that is available. +- **A notification cannot be recalled.** The test action below sends a real + notification to the configured topic. + + +## Parameter + +### Server + +The base URL of the ntfy server. The default is the public service; a self-hosted ntfy is named here instead, for example - see [installing ntfy](https://docs.ntfy.sh/install/). + +- ID: `server` +- Datatype: `string` +- Default Value: `https://ntfy.sh` + + + +### Access token + +The access token to authenticate with. A public topic needs none, so this can be left empty. See [access tokens](https://docs.ntfy.sh/config/#access-tokens). + +- ID: `access_token` +- Datatype: `password` +- Default Value: `None` + + + +### Topic + +The topic the notification is sent to. An entity that names no topic of its own is sent here. A topic needs no sign-up and has no password of its own, so give it a long random suffix nobody can guess rather than a plain word - see [picking a topic](https://docs.ntfy.sh/publish/#picking-a-topic). + +- ID: `topic` +- Datatype: `string` +- Default Value: `None` + + + +### Title + +The title of the notification the task sends itself. See [message titles](https://docs.ntfy.sh/publish/#message-title). + +- ID: `title` +- Datatype: `string` +- Default Value: `None` + + + +### Message + +The text of the notification. Configuring it makes the task send that one notification and removes its input port; left empty, each notification arrives on that port instead. + +- ID: `message` +- Datatype: `multiline string` +- Default Value: `None` + + + +### Priority + +How urgently the notification is shown, for the one the task sends itself. See [message priority](https://docs.ntfy.sh/publish/#message-priority). + +- ID: `priority` +- Datatype: `string` +- Default Value: `default` + + + +### Notification tags + +Tags shown with the notification, as a comma separated list. A tag that names an emoji is shown as that emoji. Used for the notification the task sends itself. See [tags and emojis](https://docs.ntfy.sh/publish/#tags-emojis) and the [list of emoji names](https://docs.ntfy.sh/emojis/). + +- ID: `notification_tags` +- Datatype: `string` +- Default Value: `None` + + + +### Click URL + +The URL opened when the notification is tapped. Used for the notification the task sends itself. See [click action](https://docs.ntfy.sh/publish/#click-action). + +- ID: `click` +- Datatype: `string` +- Default Value: `None` + +## Advanced Parameter + +### Markdown + +If enabled, the message is formatted as Markdown. Note that the Android app shows the markup as typed; the web app and the desktop clients render it. See [Markdown formatting](https://docs.ntfy.sh/publish/#markdown-formatting). + +- ID: `markdown` +- Datatype: `boolean` +- Default Value: `false` + + + +### Delay + +Holds the notification back: a duration such as `30m`, a timestamp, or a time of day such as `9am`, which the server reads in its own time zone. Between ten seconds and three days. See [scheduled delivery](https://docs.ntfy.sh/publish/#scheduled-delivery). + +- ID: `delay` +- Datatype: `string` +- Default Value: `None` + + + +### Icon URL + +The URL of a PNG or JPEG shown as the notification's icon. See [icons](https://docs.ntfy.sh/publish/#icons). + +- ID: `icon` +- Datatype: `string` +- Default Value: `None` + + + +### Email copy + +An address the server also sends the notification to as mail. Needs a server configured for mail delivery. See [e-mail notifications](https://docs.ntfy.sh/publish/#e-mail-notifications). + +- ID: `email` +- Datatype: `string` +- Default Value: `None` diff --git a/docs/build/reference/customtask/cmem_plugin_pgvector-Search.md b/docs/build/reference/customtask/cmem_plugin_pgvector-Search.md index aad25c831..e8e7198d1 100644 --- a/docs/build/reference/customtask/cmem_plugin_pgvector-Search.md +++ b/docs/build/reference/customtask/cmem_plugin_pgvector-Search.md @@ -1,6 +1,6 @@ --- title: "Search Vector Embeddings" -description: "Search for top-k metadata stored in Postgres Vector Store (PGVector)." +description: "Search for the top-k nearest matches to a query embedding in Postgres Vector Store (PGVector)." icon: octicons/cross-reference-24 tags: - WorkflowTask @@ -18,19 +18,15 @@ tags: e.g. with cmemc. -This workflow task search for the top-k metadata stored into Postgres Vector Store. +This workflow task searches for the top-k nearest vectors stored in a collection in Postgres +Vector Store (PGVector). -The incoming embedding entities are used to retrieve the nearest top-k -vectors in the collection stored in the Postgres Vector Store. -It is possible to specify which paths are going to be used for searching as well as which Postgres -Vector Store and collection name. +Each incoming entity supplies a query embedding on the single input path. The task compares it +against the vectors in the configured collection and attaches the top-k nearest matches to the +entity as a JSON array on an additional output path, together with each match's metadata, its +stored embedding source, and its distance to the query. -The task uses the embeddings from the path configured with the Embedding Query Path -parameter (`embedding_query_path`, default value: `_embedding`) to search over the collection. -The results are provided in the output path configured with the Search Result Path parameter -(`search_result_path`, default value: `_search_result`). - -The results in this output are structured like this: +The result is structured like this: ``` json [ @@ -44,6 +40,8 @@ The results in this output are structured like this: ] ``` +Use together with **Store Vector Embeddings**, which populates the collection this task searches. + ## Parameter @@ -99,7 +97,7 @@ The database name. ### Collection Name -The name of the collection that will be used for search. +The name of the collection in Postgres Vector Store (PGVector). - ID: `collection_name` - Datatype: `string` @@ -119,7 +117,7 @@ The path containing the search result in the output entities. ### Embedding Query Path -The path containing the embedding to be used for searching. +The path containing the query embedding to search with. Its value must be a string representation of a list of floats, e.g. `[0.1, 0.2, ...]`; the task fails if it cannot be parsed as one. - ID: `embedding_query_path` - Datatype: `string` @@ -139,7 +137,7 @@ The number of entries to be returned in the search result. ### Distance Strategy -The distance strategy to use. (default: COSINE) +The distance strategy used to rank search matches. - ID: `distance_strategy` - Datatype: `enumeration` diff --git a/docs/build/reference/customtask/cmem_plugin_pgvector-Store.md b/docs/build/reference/customtask/cmem_plugin_pgvector-Store.md index 4371e6f68..a4a090866 100644 --- a/docs/build/reference/customtask/cmem_plugin_pgvector-Store.md +++ b/docs/build/reference/customtask/cmem_plugin_pgvector-Store.md @@ -18,12 +18,19 @@ tags: e.g. with cmemc. -This plugin workflow store embeddings into Postgres Vector Store. +This workflow task stores embeddings and their metadata into a collection in Postgres Vector +Store (PGVector). -The vector embeddings and its respective metadata are going to be stored into a collection inside -the Postgres Vector Store. -It is possible to specify either the name of the attributes containing the vectors as well as the -metadata. +The task accepts entities of any schema on its input port. Two of the incoming paths supply the +text and the embedding to store; every other path, or a fixed subset of them, is stored alongside +as metadata. The task produces no output port: it is a terminal step and hands nothing on to +later tasks in the workflow. + +Use together with **Search Vector Embeddings**, which reads back the collection this task +populates. + +If the incoming entities have no path configured for the text or the embedding, the task fails +before storing anything. ## Parameter @@ -80,7 +87,7 @@ The database name. ### Collection Name -The name of the collection that will be used for search. +The name of the collection in Postgres Vector Store (PGVector). - ID: `collection_name` - Datatype: `string` @@ -90,7 +97,7 @@ The name of the collection that will be used for search. ### Pre Delete Collection -If set to true, then the collection will removed at the beginning. +If enabled, deletes any existing collection with the same name before storing, discarding its previous contents. If disabled, new vectors are added to whatever the collection already contains. - ID: `pre_delete_collection` - Datatype: `boolean` @@ -110,7 +117,7 @@ The name of the path to use for reading the embedding source. ### Embedding Path -The name of the path to use for reading the embeddings. +The name of the path to use for reading the embeddings. Its value must be a string representation of a list of floats, e.g. `[0.1, 0.2, ...]`. - ID: `embedding_path` - Datatype: `string` @@ -120,7 +127,7 @@ The name of the path to use for reading the embeddings. ### Metadata Paths -The comma separated list path names to be used as metadata. Empty name means all paths (except embedding source and embedding) will be used +The comma-separated list of path names to store as metadata. An empty value means every path except the source and embedding paths is used. - ID: `metadata_paths` - Datatype: `string` @@ -130,7 +137,7 @@ The comma separated list path names to be used as metadata. Empty name means all ### Batch Processing Size -The number of entries to be processed in batch. +The number of entries to accumulate before writing them to the database as one batch. - ID: `batch_processing_size` - Datatype: `Long` diff --git a/docs/build/reference/customtask/cmem_plugin_querycatalog-ExecuteCatalogQuery.md b/docs/build/reference/customtask/cmem_plugin_querycatalog-ExecuteCatalogQuery.md new file mode 100644 index 000000000..e2ef9781e --- /dev/null +++ b/docs/build/reference/customtask/cmem_plugin_querycatalog-ExecuteCatalogQuery.md @@ -0,0 +1,118 @@ +--- +title: "Execute Catalog Query" +description: "Run a query stored in a query catalog and hand on its result." +icon: octicons/cross-reference-24 +tags: + - WorkflowTask + - PythonPlugin +--- + +# Execute Catalog Query + + + +!!! note inline end "Python Plugin" + + This operator is part of a Python Plugin Package. + In order to use it, you need to install it, + e.g. with cmemc. + + +This task runs a query that is stored in a query catalog, so a workflow references the +stored definition instead of carrying a second copy of it. + +## What flows through the ports + +What leaves the task follows the type of the query. A SELECT query produces one entity +per result row with one value per projected variable, and a variable the query left +unbound produces no value rather than an empty string. An ASK query produces a single +entity carrying `true` or `false`. A CONSTRUCT or DESCRIBE query produces its +statements as quad entities with no graph set, ready for any task that consumes quads. +An update query produces nothing at all: the task is a terminal step and hands nothing +on. + +A query that contains placeholders turns their names into input fields, so the +preceding task can be mapped onto them, and the query is then run once per incoming +entity with the results of all of them arriving together. A query without placeholders +has no input and runs on its own. + +## Where the task sits + +**Get Catalog Query** and **List Catalog Queries** describe queries without running +them, and are the way to find the query this task should be pointed at. + +## Caveats + +- **The stored query is read again on every execution.** Editing it in the query + catalog changes what runs next time, which is the point of the task - but it also + means a query that gained a placeholder since the workflow was wired fails, naming + what changed, rather than running with a hole in it. +- **The query kind is a safety gate, not only a filter.** A query that has become an + update is refused while the task is set to read queries, so editing the catalog + cannot turn a reading step into a writing one. +- **The whole result is held in memory.** The store sends it in one response, so a + query returning very many rows is limited by what the DataIntegration process can + hold. +- **Continuing past a failed input entity can leave a write half applied.** Entities + are run one after another against the store and nothing rolls an earlier one back. + + +## Parameter + +### Catalog Graph + +The graph the query is read from. The list offers the query catalogs first, then the shape catalogs, then any other graph that holds queries, with the number of read and write queries each one contains. + +- ID: `catalog_graph` +- Datatype: `string` +- Default Value: `None` + + + +### Query Kind + +Which half of the catalog the query list offers. It also decides what this task is allowed to run: a query that has become an update since the task was configured is refused while this is set to read queries. + +- ID: `query_kind` +- Datatype: `string` +- Default Value: `read` + + + +### Query + +The query to use, picked by its label. Choose the catalog graph and the query kind first; changing either of them clears this field. + +- ID: `query` +- Datatype: `string` +- Default Value: `None` + + + +### Context + + + +- ID: `context` +- Datatype: `PluginContext` +- Default Value: `None` + +## Advanced Parameter + +### Timeout + +How many milliseconds the store may spend on one execution before it is aborted. Zero leaves the decision to the store, which is the right choice unless a query has been seen to run away. + +- ID: `timeout` +- Datatype: `Long` +- Default Value: `0` + + + +### On error + +What happens when the query fails for a single input entity. Either way the workflow ends in an error. Note that continuing through a write query can leave part of its changes applied: the entities are run one after another and nothing rolls them back. + +- ID: `on_error` +- Datatype: `string` +- Default Value: `stop` diff --git a/docs/build/reference/customtask/cmem_plugin_querycatalog-GetCatalogQuery.md b/docs/build/reference/customtask/cmem_plugin_querycatalog-GetCatalogQuery.md new file mode 100644 index 000000000..9c4a939fe --- /dev/null +++ b/docs/build/reference/customtask/cmem_plugin_querycatalog-GetCatalogQuery.md @@ -0,0 +1,76 @@ +--- +title: "Get Catalog Query" +description: "Emit one entity describing a query stored in a query catalog." +icon: octicons/cross-reference-24 +tags: + - WorkflowTask + - PythonPlugin +--- + +# Get Catalog Query + + + +!!! note inline end "Python Plugin" + + This operator is part of a Python Plugin Package. + In order to use it, you need to install it, + e.g. with cmemc. + + +This task looks a query up in a query catalog and emits one entity describing it, +without running it. + +The entity carries the catalog graph, the query IRI, its label and description, its +type, its query text as stored, and the names of the placeholders that text contains. +The query text is emitted unfilled, with its placeholders intact, which is what makes +it useful to a task that wants to inspect or transform a query rather than execute one. + +With a query chosen on the task, it emits exactly one entity and takes no input. With +the query left empty, the queries to look up arrive on an input port instead, as a +catalog graph and a query IRI per entity, and one entity is emitted for each. An +incoming entity that leaves the catalog graph empty falls back to the catalog chosen +on the task, so a list of query IRIs from one catalog needs no constant column. + +It emits exactly what **List Catalog Queries** emits, in the same order, and the +first two values it emits are what it accepts on its own input port - so the two tasks +chain directly. **Execute Catalog Query** is the one that runs a query. + +A query IRI that the named catalog does not hold fails the task rather than being +skipped, because a workflow asking for a specific query and silently receiving nothing +is the harder failure to notice. + + +## Parameter + +### Catalog Graph + +The graph the query is read from. The list offers the query catalogs first, then the shape catalogs, then any other graph that holds queries, with the number of read and write queries each one contains. + +- ID: `catalog_graph` +- Datatype: `string` +- Default Value: `None` + + + +### Query Kind + +Which half of the catalog the query list offers. + +- ID: `query_kind` +- Datatype: `string` +- Default Value: `all` + + + +### Query + +The query to use, picked by its label. Choose the catalog graph and the query kind first; changing either of them clears this field. + +- ID: `query` +- Datatype: `string` +- Default Value: `None` + +## Advanced Parameter + +`None` diff --git a/docs/build/reference/customtask/cmem_plugin_querycatalog-ListCatalogQueries.md b/docs/build/reference/customtask/cmem_plugin_querycatalog-ListCatalogQueries.md new file mode 100644 index 000000000..d94dcbdaa --- /dev/null +++ b/docs/build/reference/customtask/cmem_plugin_querycatalog-ListCatalogQueries.md @@ -0,0 +1,60 @@ +--- +title: "List Catalog Queries" +description: "Emit one entity per query stored in a query catalog." +icon: octicons/cross-reference-24 +tags: + - WorkflowTask + - PythonPlugin +--- + +# List Catalog Queries + + + +!!! note inline end "Python Plugin" + + This operator is part of a Python Plugin Package. + In order to use it, you need to install it, + e.g. with cmemc. + + +This task reads a query catalog and emits one entity for each query it holds, +describing the query rather than running it. + +Each entity carries the catalog graph, the query IRI, its label and description, its +type, its query text as stored, and the names of the placeholders that text contains. +The query text is emitted unfilled, with its placeholders intact. The task takes no +input, so it can open a workflow. + +It emits exactly what **Get Catalog Query** emits, in the same order, so one can +replace the other without rewiring whatever consumes it. Use this task to take stock of +a catalog, to drive a loop over its queries, or to find the query IRI that +**Execute Catalog Query** should be pointed at. + +A catalog holding no query of the selected kind yields no entities and no error, which +is a legitimate answer about an empty catalog rather than a failure. + + +## Parameter + +### Catalog Graph + +The graph the query is read from. The list offers the query catalogs first, then the shape catalogs, then any other graph that holds queries, with the number of read and write queries each one contains. + +- ID: `catalog_graph` +- Datatype: `string` +- Default Value: `None` + + + +### Query Kind + +Which half of the catalog the query list offers. + +- ID: `query_kind` +- Datatype: `string` +- Default Value: `all` + +## Advanced Parameter + +`None` diff --git a/docs/build/reference/customtask/cmem_plugin_reason-plugin_reason-ReasonPlugin.md b/docs/build/reference/customtask/cmem_plugin_reason-plugin_reason-ReasonPlugin.md index ec46140aa..b71311a83 100644 --- a/docs/build/reference/customtask/cmem_plugin_reason-plugin_reason-ReasonPlugin.md +++ b/docs/build/reference/customtask/cmem_plugin_reason-plugin_reason-ReasonPlugin.md @@ -17,18 +17,22 @@ tags: In order to use it, you need to install it, e.g. with cmemc. -A task performing OWL reasoning. With an OWL ontology and a data graph as input the reasoning result is written to a specified graph. +A task performing OWL reasoning. With an OWL ontology and/or a data graph as input the reasoning result is written to a +specified graph. At least one of the two input graphs must be specified. With two different input graphs, the data +graph is reasoned with the ontology imported; with only one, or the same graph in both, that graph is reasoned on its +own. ## Options ### Data graph IRI The IRI of the input data graph. The graph IRI is selected from a list of graphs of types `di:Dataset`, `void:Dataset` -and `owl:Ontology`. +and `owl:Ontology`. Optional if an ontology graph is specified. ### Ontology graph IRI -The IRI of the input ontology graph. The graph IRI is selected from a list of graphs of type`owl:Ontology`. +The IRI of the input ontology graph. The graph IRI is selected from a list of graphs of type `owl:Ontology`. Optional if +a data graph is specified. ### Ignore missing imports @@ -172,6 +176,7 @@ If there are classes `Person`, `Student` and `Professor` as wells as the object ### Validate OWL2 profiles Validate the input ontology against OWL profiles (DL, EL, QL, RL, and Full) and annotate the result graph. +Needs an ontology graph. ### Process valid OWL profiles from input @@ -182,7 +187,7 @@ profiles validation is not done in the plugin. The valid profiles input is a com ### Output graph import Add the triple owl:imports into the output graph or add the triple - owl:imports into the ontology graph. + owl:imports into the ontology graph. Needs an ontology graph. ### Maximum RAM Percentage @@ -195,7 +200,7 @@ Maximum heap size for the Java virtual machine in the DI container running the r ### Data graph IRI -The IRI of the input data graph. +The IRI of the input data graph. At least one of data graph and ontology graph must be specified. - ID: `data_graph_iri` - Datatype: `scheme:string` @@ -205,7 +210,7 @@ The IRI of the input data graph. ### Ontology graph IRI -The IRI of the input ontology graph. +The IRI of the input ontology graph. At least one of data graph and ontology graph must be specified. - ID: `ontology_graph_iri` - Datatype: `scheme:string` @@ -375,7 +380,7 @@ Validate the input ontology against OWL profiles (DL, EL, QL, RL, and Full) and ### Output graph import -Add the triple owl:imports to the output graph or add the triple owl:imports to the ontology graph. +Add the triple owl:imports to the output graph or add the triple owl:imports to the ontology graph. Needs an ontology graph. - ID: `imports` - Datatype: `string` diff --git a/docs/build/reference/customtask/cmem_plugin_reason-plugin_validate-ValidatePlugin.md b/docs/build/reference/customtask/cmem_plugin_reason-plugin_validate-ValidatePlugin.md index 279502168..6862dd281 100644 --- a/docs/build/reference/customtask/cmem_plugin_reason-plugin_validate-ValidatePlugin.md +++ b/docs/build/reference/customtask/cmem_plugin_reason-plugin_validate-ValidatePlugin.md @@ -56,6 +56,10 @@ The IRI of the output graph for the reasoning result. ⚠️ Existing graphs will be overwritten. +The output graph holds the explanation axioms plus the validation result and is declared a +`void:Dataset`. It is not declared an `owl:Ontology`: it does not contain an ontology of its +own, it describes the validated one. + ### Write markdown explanation file If enabled, an explanation markdown file is written to the project. diff --git a/docs/build/reference/customtask/cmem_plugin_s3-Download.md b/docs/build/reference/customtask/cmem_plugin_s3-Download.md new file mode 100644 index 000000000..a5a8d9b77 --- /dev/null +++ b/docs/build/reference/customtask/cmem_plugin_s3-Download.md @@ -0,0 +1,201 @@ +--- +title: "Download S3 objects" +description: "Download objects from a bucket in S3 or S3-compatible object storage." +icon: octicons/cross-reference-24 +tags: + - WorkflowTask + - PythonPlugin +--- + +# Download S3 objects + + + +!!! note inline end "Python Plugin" + + This operator is part of a Python Plugin Package. + In order to use it, you need to install it, + e.g. with cmemc. + + +Downloads objects from a bucket into the workflow. + +## Ports + +Objects leave the task as file entities, written into a temporary directory of +the DataIntegration node. They are not added to the project, so anything that is +meant to be kept has to be written somewhere by a following task. + +The task has two mutually exclusive shapes. With a bucket configured on it, it +selects objects itself and takes no input. With the bucket left empty, the +objects to fetch arrive on an input port instead, in the form **List S3 objects** +emits, and the settings that would narrow a selection down have nothing left to +act on - the task in front has already decided. + +## Where the task sits + +**List S3 objects** produces the listing this task accepts, so a workflow can +list, narrow the result down and download only what is left. **Upload files to +S3** covers the other direction, and the file entities this task emits are what +every file consuming task in the fleet accepts. + +## Caveats + +- **Each object is written into a numbered directory of its own**, so two keys + ending in the same name do not overwrite each other. Only the name is kept, + not the key's path, and a following task therefore sees the name rather than + the key. +- **An object replaced since it was listed is refused, not fetched.** The entity + tag from the listing is sent with the request, so the workflow downloads the + bytes it selected or fails - it never silently downloads newer ones. An entity + reaching the input port without an entity tag is fetched unconditionally. +- **The error handling decides whether one failure ends the run**, but the run + ends in an error either way. Continuing only means the remaining objects are + fetched first. +- **The downloaded files are not cleaned up by the task.** The temporary + directory it writes into stays on the node until whatever clears its temporary + storage does, so a workflow fetching more than the node's disk holds will fill + it, and fetching the same bucket repeatedly accumulates copies. +- **The preview stops after ten objects**, so it can miss an object further down + that a real run would fail on. + + +## Parameter + +### Endpoint URL + +The address of the service, e.g. `https://minio.example.org`. Leave empty for Amazon S3, which is then addressed by region. + +- ID: `endpoint_url` +- Datatype: `string` +- Default Value: `None` + + + +### Region + +The region the bucket lives in. Amazon S3 needs the bucket's own region; most other implementations accept any value and ignore it. + +- ID: `region` +- Datatype: `string` +- Default Value: `us-east-1` + + + +### Credentials + +Where the credentials come from. + +- ID: `credentials` +- Datatype: `string` +- Default Value: `explicit` + + + +### Access key ID + +The access key to sign requests with. + +- ID: `access_key_id` +- Datatype: `string` +- Default Value: `None` + + + +### Secret access key + +The secret belonging to the access key. + +- ID: `secret_access_key` +- Datatype: `password` +- Default Value: `None` + + + +### Bucket + +The bucket the objects are downloaded from. Leaving it empty replaces the task's own selection with an input port. + +- ID: `bucket` +- Datatype: `string` +- Default Value: `None` + + + +### Key prefix + +Downloads only the objects whose key starts with this, e.g. `raw/2026/`. The service applies it. Empty means the whole bucket. + +- ID: `prefix` +- Datatype: `string` +- Default Value: `None` + + + +### Regular expression + +Downloads only the objects whose key matches this regular expression, e.g. `\.csv$`. It is matched against the whole key rather than against the name alone, and it has to match somewhere in it rather than all of it. Empty means every key. + +- ID: `regex` +- Datatype: `string` +- Default Value: `None` + + + +### Top level only + +If enabled, only the keys directly below the prefix are considered and everything nested deeper is folded away. The service does the folding, on a `/` delimiter. + +- ID: `top_level_only` +- Datatype: `boolean` +- Default Value: `false` + + + +### Limit + +The maximum number of objects to download, or 0 for no limit. A run that stops at the limit says so in its report. + +- ID: `limit` +- Datatype: `Long` +- Default Value: `0` + +## Advanced Parameter + +### Session token + +The session token of a temporary credential. Needed only for keys issued by a token service, which expire. + +- ID: `session_token` +- Datatype: `password` +- Default Value: `None` + + + +### Bucket addressing style + +How the bucket is named in a request. + +- ID: `addressing_style` +- Datatype: `string` +- Default Value: `auto` + + + +### Verify TLS connection + +If enabled, the certificate of the service is verified. Disable only when connecting to a service with a self-signed or otherwise untrusted certificate. + +- ID: `ssl_verify` +- Datatype: `boolean` +- Default Value: `true` + + + +### On error + +What happens when a single object cannot be transferred. Either way the workflow ends in an error. + +- ID: `on_error` +- Datatype: `string` +- Default Value: `stop` diff --git a/docs/build/reference/customtask/cmem_plugin_s3-List.md b/docs/build/reference/customtask/cmem_plugin_s3-List.md new file mode 100644 index 000000000..ff4c9a285 --- /dev/null +++ b/docs/build/reference/customtask/cmem_plugin_s3-List.md @@ -0,0 +1,184 @@ +--- +title: "List S3 objects" +description: "List the objects of a bucket in S3 or S3-compatible object storage." +icon: octicons/cross-reference-24 +tags: + - WorkflowTask + - PythonPlugin +--- + +# List S3 objects + + + +!!! note inline end "Python Plugin" + + This operator is part of a Python Plugin Package. + In order to use it, you need to install it, + e.g. with cmemc. + + +Lists the objects of a bucket, together with the metadata the service reports for +them. + +## Ports + +The task takes no input. It emits one entity per object, carrying the `bucket` it +was found in, its full `key`, the `name` that key ends in, its `size` in bytes, +its `etag`, the `last_modified` timestamp and the `storage_class`. Keys ending in +a slash are folder markers rather than content and are left out. + +## Where the task sits + +**Download S3 objects** accepts these entities on its input port, so a workflow +can list first, narrow the result down with transformations in between, and fetch +only what is left. The `etag` travels with each entity and is what lets the +download notice an object that changed in the meantime. + +Listing is also a way to write an inventory of a bucket into a Knowledge Graph +without moving any of its content. + +## Caveats + +- **The listing streams.** Entities are produced while the service is still being + paged through, so a bucket with millions of keys does not have to be held in + memory - but neither is the total known in advance, and the entity count in the + report grows as the run proceeds. +- **A regular expression is applied after the fact.** The key prefix is the only + filter the service itself applies; everything the expression rejects has + already been listed and paid for. +- **The preview stops after ten objects.** It therefore never reaches an object + further down that a real run would reject or fail on. +- **An object's size and timestamp are the service's.** They are written down as + reported, not checked against the bytes. + + +## Parameter + +### Endpoint URL + +The address of the service, e.g. `https://minio.example.org`. Leave empty for Amazon S3, which is then addressed by region. + +- ID: `endpoint_url` +- Datatype: `string` +- Default Value: `None` + + + +### Region + +The region the bucket lives in. Amazon S3 needs the bucket's own region; most other implementations accept any value and ignore it. + +- ID: `region` +- Datatype: `string` +- Default Value: `us-east-1` + + + +### Credentials + +Where the credentials come from. + +- ID: `credentials` +- Datatype: `string` +- Default Value: `explicit` + + + +### Access key ID + +The access key to sign requests with. + +- ID: `access_key_id` +- Datatype: `string` +- Default Value: `None` + + + +### Secret access key + +The secret belonging to the access key. + +- ID: `secret_access_key` +- Datatype: `password` +- Default Value: `None` + + + +### Bucket + +The bucket whose objects are listed. + +- ID: `bucket` +- Datatype: `string` +- Default Value: `None` + + + +### Key prefix + +Lists only the objects whose key starts with this, e.g. `raw/2026/`. The service applies it, so it is the cheapest way to narrow a large bucket down. Empty means the whole bucket. + +- ID: `prefix` +- Datatype: `string` +- Default Value: `None` + + + +### Regular expression + +Lists only the objects whose key matches this regular expression, e.g. `\.csv$`. It is matched against the whole key rather than against the name alone, and it has to match somewhere in it rather than all of it. Empty means every key. + +- ID: `regex` +- Datatype: `string` +- Default Value: `None` + + + +### Top level only + +If enabled, only the keys directly below the prefix are considered and everything nested deeper is folded away. The service does the folding, on a `/` delimiter. + +- ID: `top_level_only` +- Datatype: `boolean` +- Default Value: `false` + + + +### Limit + +The maximum number of objects to list, or 0 for no limit. A run that stops at the limit says so in its report. + +- ID: `limit` +- Datatype: `Long` +- Default Value: `0` + +## Advanced Parameter + +### Session token + +The session token of a temporary credential. Needed only for keys issued by a token service, which expire. + +- ID: `session_token` +- Datatype: `password` +- Default Value: `None` + + + +### Bucket addressing style + +How the bucket is named in a request. + +- ID: `addressing_style` +- Datatype: `string` +- Default Value: `auto` + + + +### Verify TLS connection + +If enabled, the certificate of the service is verified. Disable only when connecting to a service with a self-signed or otherwise untrusted certificate. + +- ID: `ssl_verify` +- Datatype: `boolean` +- Default Value: `true` diff --git a/docs/build/reference/customtask/cmem_plugin_s3-Upload.md b/docs/build/reference/customtask/cmem_plugin_s3-Upload.md new file mode 100644 index 000000000..3cbab9feb --- /dev/null +++ b/docs/build/reference/customtask/cmem_plugin_s3-Upload.md @@ -0,0 +1,175 @@ +--- +title: "Upload files to S3" +description: "Upload files to a bucket in S3 or S3-compatible object storage." +icon: octicons/cross-reference-24 +tags: + - WorkflowTask + - PythonPlugin +--- + +# Upload files to S3 + + + +!!! note inline end "Python Plugin" + + This operator is part of a Python Plugin Package. + In order to use it, you need to install it, + e.g. with cmemc. + + +Writes the files that arrive on its input port into a bucket. + +## Ports + +Files arrive on the input port, which is what a task in front of this one is +for - a download task, a task that produced a file, or anything else emitting +file entities. The task emits one entity per file it dealt with, carrying the +`bucket` and the `key` it wrote, the `name` the key ends in, the `size`, the +`etag` the service assigned and whether the file was `uploaded` or `skipped`. + +That entity tag is a checksum of what now sits in the bucket, so a workflow can +write it into a Knowledge Graph next to whatever else it records about the file. + +## Where the task sits + +**Download S3 objects** and every other download task in the fleet produce the +file entities this task accepts, which is how a workflow copies from one system +into object storage. **List S3 objects** reads a bucket back afterwards. + +## Caveats + +- **The key is the prefix plus the file's name**, and the name is all that a + file entity carries - the key an object came from is not part of it. Two files + with the same name therefore resolve to the same key. The second one is + reported as a failure rather than written over the first. +- **A large file is written in parts** and never held in memory as a whole. A + file the project holds is first fetched to a temporary directory of the + DataIntegration node, so the node needs room for it. +- **Skip and Fail ask the service whether the key is taken** before writing, one + request per file, and something writing that key in between is not noticed. + Replace asks nothing and simply writes. +- **The error handling decides whether one failure ends the run**, but the run + ends in an error either way. Continuing only means the remaining files are + written first. + + +## Parameter + +### Endpoint URL + +The address of the service, e.g. `https://minio.example.org`. Leave empty for Amazon S3, which is then addressed by region. + +- ID: `endpoint_url` +- Datatype: `string` +- Default Value: `None` + + + +### Region + +The region the bucket lives in. Amazon S3 needs the bucket's own region; most other implementations accept any value and ignore it. + +- ID: `region` +- Datatype: `string` +- Default Value: `us-east-1` + + + +### Credentials + +Where the credentials come from. + +- ID: `credentials` +- Datatype: `string` +- Default Value: `explicit` + + + +### Access key ID + +The access key to sign requests with. + +- ID: `access_key_id` +- Datatype: `string` +- Default Value: `None` + + + +### Secret access key + +The secret belonging to the access key. + +- ID: `secret_access_key` +- Datatype: `password` +- Default Value: `None` + + + +### Bucket + +The bucket the files are written to. It has to exist already. + +- ID: `bucket` +- Datatype: `string` +- Default Value: `None` + + + +### Key prefix + +Written in front of each file's name to make the key, e.g. `incoming/`. Empty writes the names to the top of the bucket. + +- ID: `prefix` +- Datatype: `string` +- Default Value: `None` + + + +### When the key is already taken + +What happens to an object the bucket already holds under the key this task would write. Replacing is what makes a repeated run leave the bucket as it was after the first one. + +- ID: `on_existing` +- Datatype: `string` +- Default Value: `replace` + +## Advanced Parameter + +### Session token + +The session token of a temporary credential. Needed only for keys issued by a token service, which expire. + +- ID: `session_token` +- Datatype: `password` +- Default Value: `None` + + + +### Bucket addressing style + +How the bucket is named in a request. + +- ID: `addressing_style` +- Datatype: `string` +- Default Value: `auto` + + + +### Verify TLS connection + +If enabled, the certificate of the service is verified. Disable only when connecting to a service with a self-signed or otherwise untrusted certificate. + +- ID: `ssl_verify` +- Datatype: `boolean` +- Default Value: `true` + + + +### On error + +What happens when a single object cannot be transferred. Either way the workflow ends in an error. + +- ID: `on_error` +- Datatype: `string` +- Default Value: `stop` diff --git a/docs/build/reference/customtask/cmem_plugin_salesforce-SoqlQuery.md b/docs/build/reference/customtask/cmem_plugin_salesforce-SoqlQuery.md index 926d7e68e..3a46910d1 100644 --- a/docs/build/reference/customtask/cmem_plugin_salesforce-SoqlQuery.md +++ b/docs/build/reference/customtask/cmem_plugin_salesforce-SoqlQuery.md @@ -31,18 +31,19 @@ SOQL uses the SELECT statement combined with filtering statements to return sets data, which can optionally be ordered. For a complete description of the syntax, see [Salesforce SOQL SELECT Syntax](https://developer.salesforce.com/docs/atlas.en-us.soql_sosl.meta/soql_sosl/sforce_api_calls_soql_select.htm). -In the Advanced Options section, you can enable / disable the validation of your -SOQL Query. By default, this Parse SOQL option is set `True` (enabled). +This task ignores any entities it receives; the query comes only from the SOQL Query +parameter. A query that matches no records fails with an error rather than returning +an empty result. Examples: -Retrieve all standard fields from all Lead resources. (without parser validation) +Retrieve all standard fields from all Lead resources. ```text SELECT FIELDS(STANDARD) FROM Lead ``` -Retrieve first name and last name of all Contact resources. (with parser validation) +Retrieve first name and last name of all Contact resources. ```text SELECT Contact.Firstname, Contact.Lastname FROM Contact @@ -96,7 +97,7 @@ The query text of your SOQL query. SOQL uses the SELECT statement combined with ### Dataset -In addition to have direct output of the fetched entities of your SOQL query, you can directly write the response to a JSON dataset (mostly for debugging purpose). +Besides the direct output of the fetched entities, write the complete raw query response - including Salesforce's internal metadata for each record - to a JSON dataset (mostly for debugging purposes). - ID: `dataset` - Datatype: `string` diff --git a/docs/build/reference/customtask/cmem_plugin_shapes-plugin_shapes-ShapesPlugin.md b/docs/build/reference/customtask/cmem_plugin_shapes-plugin_shapes-ShapesPlugin.md index 27864136c..6495d3ba7 100644 --- a/docs/build/reference/customtask/cmem_plugin_shapes-plugin_shapes-ShapesPlugin.md +++ b/docs/build/reference/customtask/cmem_plugin_shapes-plugin_shapes-ShapesPlugin.md @@ -17,32 +17,67 @@ tags: In order to use it, you need to install it, e.g. with cmemc. -This workflow task generates SHACL (Shapes Constraint Language) -node and property shapes by analyzing instance data from a knowledge graph. The generated -shapes describe the structure and properties of the classes used in the data graph. - -## Usage - -The plugin analyzes an input data graph and creates: - -- **Node shapes**: One for each class (`rdf:type`) used in the data graph -- **Property shapes**: For all properties associated with each class, including: - - Regular object properties (subject → object relationships) - - Inverse object properties (object ← subject relationships, marked with ← prefix) - - Datatype properties (literal values) - -## Output - -The generated shapes are written to a shape catalog graph with: - -- Unique URIs based on UUIDs (UUID5 derived from class/property IRIs) -- Human-readable labels and names (using namespace prefixes when available) -- Metadata including source data graph reference and timestamps -- Optional plugin provenance information (see advanced options) +This workflow task generates SHACL (Shapes Constraint Language) node +and property shapes by analyzing the instance data of a knowledge graph. The generated +shapes describe the classes the data uses and the properties each class is used with. + +The data graph and the shape catalog are both chosen by IRI parameter rather than by +workflow connection, so the task has neither an input nor an output port. It is a +terminal step: it reads and writes graphs directly and hands nothing on to a following +task. + +## What is generated + +- A **node shape** for every class that instances are typed with. +- A **property shape** for every property those instances use: object properties in the + subject → object direction, object properties in the object ← subject direction, whose + names carry a `←` prefix, and datatype properties, whose values are literals. + +Every shape gets an IRI derived from a UUID5 of the class or property it describes, so +within one catalog the same class or property always maps to the same shape. Names and +labels come from the title eccenca Corporate Memory resolves for that class or property. +A description follows wherever one can be resolved - the `rdfs:comment`, +`dcterms:description` or `skos:definition` of the class or property, looked up wherever +it is defined, so a vocabulary graph counts as well as the data graph. A class or +property that nothing describes gets no `sh:description`, and neither does a shape for +the object ← subject direction: the description is written about the property, so on the +inverse path it would say the opposite of what the shape holds. + +Every property shape carries `shui:showAlways`, and a shape for the object ← subject +direction carries `shui:inversePath` as well. A node shape carries the `foaf:depiction` of +its target class wherever the data graph or the class's own vocabulary offers one. The +catalog itself records the data graph it was generated from, when it was written, and the +classes it manages. + +## Caveats + +Names, labels and descriptions are requested in English. Where a vocabulary offers no +English text, Corporate Memory answers in whatever language it does have, and the +literal carries that language rather than claiming to be English. A class or property +the deployment knows nothing about falls back to a name built from its IRI, which has +no language at all and is written without a tag. + +A property used by several classes gets one property shape, shared by every node shape +that uses it. Its `sh:nodeKind` is decided by whichever of those classes sorts first, so a +property carrying IRI values under one class and literal values under another is described +as only one of the two - the same one on every run over the same data. + +A class or property that is a blank node is skipped. Instance data sharing a graph with an +ontology types things with anonymous class expressions, and a shape has nothing to target +in one. + +`sh:datatype rdf:langString` is added as soon as any value of a property carries a +language tag, however few of them do. A property whose values mix tagged and untagged +literals therefore gets a shape that its own source data does not satisfy. + +Adding to an existing catalog replaces the name, label and description of every shape the +run writes, and the classes the catalog declares, rather than leaving an earlier run's +beside them. Everything else in the catalog is left alone, including shapes this run did +not produce - that is what adding is for. ## Example -Given a data graph with: +Given a data graph typing instances with a vocabulary the deployment knows: ``` turtle ex:Person123 a ex:Person ; @@ -50,21 +85,21 @@ ex:Person123 a ex:Person ; ex:knows ex:Person456 . ``` -The plugin generates: - -- A node shape for `ex:Person` with `sh:targetClass ex:Person` +the task generates a node shape for `ex:Person` ``` turtle graph:90ee6e27-59b1-5ac8-9d7a-116c60c6791a a sh:NodeShape ; rdfs:label "Person (ex:)"@en ; sh:name "Person (ex:)"@en ; + sh:description "A person."@en ; sh:property graph:0fcf371d-f99a-5eeb-ab50-6e6b5fbb0e06 , graph:dd5c6728-75a2-5215-8a5d-f9cd4077aaea ; sh:targetClass ex:Person . ``` -- Property shapes for `ex:name` (datatype property) and `ex:knows` (object property) +together with a property shape for each of `ex:knows` and `ex:name`, of which the first +reads ``` turtle graph:0fcf371d-f99a-5eeb-ab50-6e6b5fbb0e06 a sh:PropertyShape ; @@ -81,7 +116,7 @@ graph:0fcf371d-f99a-5eeb-ab50-6e6b5fbb0e06 a sh:PropertyShape ; ### Input data graph -The knowledge graph containing the instance data to be analyzed for the SHACL shapes generation. +The knowledge graph holding the instance data to analyze. - ID: `data_graph_iri` - Datatype: `scheme:string` @@ -91,7 +126,7 @@ The knowledge graph containing the instance data to be analyzed for the SHACL sh ### Output shape catalog -The knowledge graph the generated shapes will be added to. +The graph the generated shapes are written to. - ID: `shapes_graph_iri` - Datatype: `scheme:string` @@ -99,19 +134,9 @@ The knowledge graph the generated shapes will be added to. -### Output shape catalog label - -The label for the shape catalog graph. If no label is specified for a new shapes graph, a label will be generated. If no label is specified when adding to a shapes graph, the original label will be kept, or, if the existing graph does not have a label, a label will be generated. Only labels with language tag "en" or without language tag are considered. - -- ID: `label` -- Datatype: `string` -- Default Value: `None` - - - -### Handle existing output graph +### Handle an existing shape catalog -Add result to the existing graph (add result to graph), overwrite the existing graph with the result (replace existing graph with result), or stop the workflow if the output graph already exists (stop workflow if output graph exists). +What to do when the output shape catalog already exists. - ID: `existing_graph` - Datatype: `string` @@ -119,9 +144,9 @@ Add result to the existing graph (add result to graph), overwrite the existing g -### Import the output graph into the central shapes catalog +### Import into the central shape catalog -Import the SHACL shapes graph in the CMEM shapes catalog by adding an `owl:imports` statement to the central CMEM shapes catalog. If the graph is not imported, the new shapes are not activated and used. +If enabled, the generated catalog is imported into the central shape catalog by adding an `owl:imports` statement to it. Shapes in a catalog the central one does not import are not picked up. - ID: `import_shapes` - Datatype: `boolean` @@ -129,9 +154,19 @@ Import the SHACL shapes graph in the CMEM shapes catalog by adding an `owl:impor ## Advanced Parameter +### Output shape catalog label + +The label of the shape catalog. Left empty, a new catalog gets a generated label, and a catalog being added to keeps the label it has - but a catalog being replaced is rewritten whole, generated label included. Only a label tagged "en" or carrying no language tag counts as an existing label, and only such a label is replaced. + +- ID: `label` +- Datatype: `string` +- Default Value: `None` + + + ### Fetch namespace prefixes from prefix.cc -Fetch the list of namespace prefixes from instead of using the local prefix database. If unavailable, fall back to the local database. Prefixes defined in the Corporate Memory project override database prefixes. Enabling this option exposes your IP address to prefix.cc but no other data is shared. If unsure, keep this option disabled. See . +If enabled, the namespace prefix list is fetched from instead of the local prefix database, falling back to the local database when the service cannot be reached. Prefixes defined in the project win over both. Enabling this exposes your IP address to prefix.cc; no other data is shared. If unsure, leave it disabled. See . - ID: `prefix_cc` - Datatype: `boolean` @@ -141,7 +176,7 @@ Fetch the list of namespace prefixes from instead of using t ### Properties to ignore -Provide the list of properties (as IRIs) to ignore. +The properties to leave out of the generated shapes, as IRIs, one per line. - ID: `ignore_properties` - Datatype: `multiline string` @@ -149,9 +184,9 @@ Provide the list of properties (as IRIs) to ignore. -### Types to ignore +### Classes to ignore -Provide the list of types (as IRIs) to ignore. +The classes to leave out of the generated shapes, as IRIs, one per line. - ID: `ignore_types` - Datatype: `multiline string` @@ -159,10 +194,40 @@ Provide the list of types (as IRIs) to ignore. -### Include plugin provenance +### Include task provenance -Add information about the plugin and plugin settings to the shapes graph. +If enabled, the shape catalog also records the task that generated it and the parameter values it ran with. Where the type of the task cannot be determined, nothing is recorded and the run continues with a warning in the log. - ID: `plugin_provenance` - Datatype: `boolean` - Default Value: `false` + + + +### Omit the namespace prefix from property names + +If enabled, property shape names and labels leave off the trailing namespace prefix, reading "label" rather than "label (rdfs:)". Node shape names always keep it. + +- ID: `omit_namespace_addon` +- Datatype: `boolean` +- Default Value: `false` + + + +### Declare the classes the catalog manages + +If enabled, the shape catalog states with `shui:managedClasses` which classes it manages: node shapes, property shapes, property groups, SPARQL constraints, prefix declarations, chart visualizations, table reports, widget integrations and workflow triggers. + +- ID: `managed_classes` +- Datatype: `boolean` +- Default Value: `true` + + + +### Add depictions to node shapes + +If enabled, a node shape is given the `foaf:depiction` of its target class, where one can be found. The class is looked up in the data graph and in the graph named by its own namespace, tried both with and without the trailing separator, because the graph a vocabulary was loaded into is not recorded anywhere. A class depicted more than once contributes one of its depictions, the same one on every run. + +- ID: `depictions` +- Datatype: `boolean` +- Default Value: `true` diff --git a/docs/build/reference/customtask/cmem_plugin_ssh-Download.md b/docs/build/reference/customtask/cmem_plugin_ssh-Download.md index dac2358a9..c64e080e2 100644 --- a/docs/build/reference/customtask/cmem_plugin_ssh-Download.md +++ b/docs/build/reference/customtask/cmem_plugin_ssh-Download.md @@ -1,6 +1,6 @@ --- title: "Download SSH files" -description: "Download files from a given SSH instance" +description: "Download files from an SSH server." icon: octicons/cross-reference-24 tags: - WorkflowTask @@ -18,43 +18,46 @@ tags: e.g. with cmemc. -This workflow task downloads files from a specified SSH instance. +Downloads files from an SSH server into the workflow. -By providing the hostname, username, port and authentication method, you can specify the -folder from which the data should be extracted. +What is downloaded depends on the input port. When nothing is connected, the task lists +the configured directory itself and downloads the files matching the regular +expression. When a file listing is connected, it downloads exactly the paths carried in +the `file_name` value of the incoming entities, and the directory, regular expression +and subfolder settings are then without effect. Either way the files leave the output +port as file entities, stored in a temporary directory on the eccenca Corporate Memory +host from where the following tasks read them. -You can also define a regular expression to include or exclude specific files. +**List SSH files** produces the listing this task accepts, so a workflow can list +first, narrow the result down and download only what is left. **Upload SSH files** +covers the other direction. -There is also an option to prevent files in subfolders from being included. +The **Preview results** action lists the first ten files that a run without input would +download, without running the workflow. -## Authentication Methods +## Caveats -- **Password:** Only the password will be used for authentication. The private key field is -ignored, even if filled. -- **Key:** The private key will be used for authentication. If the key is encrypted, the password -will be used to decrypt it. +- Only the plain file name of a remote file is kept. Downloaded files of the same name +coming from different folders overwrite each other, and just one of them reaches the +output port. +- A file that cannot be read is dropped from the output silently with both Ignore and +Warning. The warning names the unreadable files seen while listing, which is not -### Error handling modes +necessarily the same set of files. -- **Ignore:** Ignores the permission rights of files and lists downloads all files it has access to. -Skips folders and files when there is no correct permission. -- **Warning:** Warns the user about files that the user has no permission rights to. Downloads -all other files and skips files folder when there is no correct permission. -- **Error:** Throws an error when there is a single file or folder with incorrect permission rights. - -#### Note - -- If a connection cannot be established within 20 seconds, a timeout occurs. -- Currently supported key types are: RSA, ECDSA, Ed25519. -- Setting the maximum amount of workers to more than 1 may cause a Channel Exception when -the amount of files is too large +- A folder that cannot be read is skipped silently unless the error handling is set to +Error, and the files below it are then never downloaded. +- The task logs in with the configured credentials only, and offers no key of the +machine it runs on. The host key of the server in turn is accepted as presented and +never checked against a known hosts list. +- Establishing the connection fails after 20 seconds. ## Parameter ### Hostname -Hostname to connect to. Usually in the form of an IP address +Host name or IP address of the SSH server. - ID: `hostname` - Datatype: `string` @@ -64,7 +67,7 @@ Hostname to connect to. Usually in the form of an IP address ### Port -The port on which the connection will be tried on. Default is 22. +TCP port the SSH server listens on. - ID: `port` - Datatype: `Long` @@ -74,7 +77,7 @@ The port on which the connection will be tried on. Default is 22. ### Username -The username with which a connection will be instantiated. +Account to log in as. - ID: `username` - Datatype: `string` @@ -84,7 +87,7 @@ The username with which a connection will be instantiated. ### Authentication method -The method that is used to connect to the SSH server. +How the task authenticates against the server. - ID: `authentication_method` - Datatype: `string` @@ -94,7 +97,7 @@ The method that is used to connect to the SSH server. ### Private key -Your private key to connect via SSH. +Private key in PEM format, used when the authentication method is Key. RSA, ECDSA and Ed25519 keys are supported. - ID: `private_key` - Datatype: `password` @@ -104,7 +107,7 @@ Your private key to connect via SSH. ### Password -Depending on your authentication method this will either be used toconnect via password to SSH, or to decrypt the SSH private key +Password of the account, or the passphrase of the private key when the authentication method is Key. - ID: `password` - Datatype: `password` @@ -114,7 +117,7 @@ Depending on your authentication method this will either be used toconnect via p ### Path -The currently selected path within your SSH instance. Auto-completion starts from user home folder, use '..' for parent directory or '/' for root directory. +Remote directory the files are downloaded from, and ignored when a file listing arrives on the input port. Autocompletion starts in the home directory of the account, use '..' for the parent directory or '/' for the root directory. - ID: `path` - Datatype: `string` @@ -122,9 +125,9 @@ The currently selected path within your SSH instance. Auto-completion starts fro -### Error handling for missing permissions +### Error handling -A choice on how to handle errors concerning the permissions rights.When choosing 'ignore' all files get skipped if the current user has correct permission rights.When choosing 'warning' all files get downloaded however there will be a mention that some of the files are not under the users permissionsif there are any and these get skipped.When choosing 'error' the files will not get downloaded if thereis even a single file the user has no access to. +How the task reacts to files and folders the account cannot read. - ID: `error_handling` - Datatype: `string` @@ -134,7 +137,7 @@ A choice on how to handle errors concerning the permissions rights.When choosing ### No subfolder -When this flag is set, only files from the current directory will be downloaded. +If enabled, only the given directory is searched and its subfolders are left out. - ID: `no_subfolder` - Datatype: `boolean` @@ -144,7 +147,7 @@ When this flag is set, only files from the current directory will be downloaded. ### Regular expression -A regular expression used to define which files will get downloaded. +Regular expression a file name has to match completely to be downloaded. It is matched against the plain file name, not against the path. - ID: `regex` - Datatype: `string` @@ -152,9 +155,9 @@ A regular expression used to define which files will get downloaded. ## Advanced Parameter -### Maximum amount of workers +### Maximum number of workers -Determines the amount of workers used for concurrent thread execution of the task. Default is 1, maximum is 32. Note that too many workers can cause a ChannelException. +Number of threads the Preview results action uses to list subfolders in parallel, from 1 to 32. A workflow run always lists with a single thread. Too many parallel channels make some servers refuse them with a ChannelException. - ID: `max_workers` - Datatype: `Long` diff --git a/docs/build/reference/customtask/cmem_plugin_ssh-Execute.md b/docs/build/reference/customtask/cmem_plugin_ssh-Execute.md index 30174ed39..f2b05c2b7 100644 --- a/docs/build/reference/customtask/cmem_plugin_ssh-Execute.md +++ b/docs/build/reference/customtask/cmem_plugin_ssh-Execute.md @@ -1,6 +1,6 @@ --- title: "Execute commands via SSH" -description: "Execute commands on a given SSH instance." +description: "Execute a command on an SSH server." icon: octicons/cross-reference-24 tags: - WorkflowTask @@ -18,45 +18,41 @@ tags: e.g. with cmemc. -This workflow task executes commands on a given SSH instance. +Runs a command on an SSH server and collects what it produces. -By providing the hostname, username, port and authentication method, you can specify the -folder in which the command should be executed in. +The input and output methods decide the shape of the task. With file input, an input +port accepts file entities and the command runs once per incoming file, with the +content of that file on its standard input; without it, the command runs exactly once. +What leaves the task is either one entity per run, carrying the exit code, the standard +output and the standard error, or one file per run holding the raw standard output, or +nothing at all, in which case the task ends the branch of the workflow it sits in. -## Input Methods +**Upload SSH files** and **Download SSH files** move the files a command works on, so a +common chain is to upload files, run a command over them and download the result. -- **No input:** The command will be executed with no input attached to the plugin. Stdin -is non-existent in this case. -- **File input:** The command will be executed with the stdin being represented by the -files that are connected via the input port of the plugin. This also allows for looping -over multiple files executing the same command over them. +## Caveats - -### Output Methods - -- **Structured process output:** The output will produce entities with its own schema including -the stdout and stderr as well as the exit code to confirm the execution of the command. -- **File output:** The stdout will be converted into a file a be provided for further use. -- **No output:** The output port will be closed. - -#### Authentication Methods - -- **Password:** Only the password will be used for authentication. The private key field is -ignored, even if filled. -- **Key:** The private key will be used for authentication. If the key is encrypted, the password -will be used to decrypt it. - -#### Note - -- If a connection cannot be established within 20 seconds, a timeout occurs. -- Currently supported key types are: RSA, ECDSA, Ed25519. +- The exit code is never inspected. A failing command leaves the task successful and +the workflow running, and the failure shows up only in the emitted exit code and +standard error. With file output, both are dropped and the standard output is kept +alone. +- The command is sent as it is written and interpreted by the login shell of the +account, without any escaping or checking, and it runs with every right that account +has. +- The command runs in whatever directory the account lands in on login. The configured +directory is not entered. +- With file input, each incoming file is read into memory as a whole before it is sent. +- The task logs in with the configured credentials only, and offers no key of the +machine it runs on. The host key of the server in turn is accepted as presented and +never checked against a known hosts list. +- Establishing the connection fails after 20 seconds. ## Parameter ### Hostname -Hostname to connect to. Usually in the form of an IP address +Host name or IP address of the SSH server. - ID: `hostname` - Datatype: `string` @@ -66,7 +62,7 @@ Hostname to connect to. Usually in the form of an IP address ### Port -The port on which the connection will be tried on. Default is 22. +TCP port the SSH server listens on. - ID: `port` - Datatype: `Long` @@ -76,7 +72,7 @@ The port on which the connection will be tried on. Default is 22. ### Username -The username with which a connection will be instantiated. +Account to log in as. - ID: `username` - Datatype: `string` @@ -86,7 +82,7 @@ The username with which a connection will be instantiated. ### Authentication method -The method that is used to connect to the SSH server. +How the task authenticates against the server. - ID: `authentication_method` - Datatype: `string` @@ -96,7 +92,7 @@ The method that is used to connect to the SSH server. ### Private key -Your private key to connect via SSH. +Private key in PEM format, used when the authentication method is Key. RSA, ECDSA and Ed25519 keys are supported. - ID: `private_key` - Datatype: `password` @@ -106,7 +102,7 @@ Your private key to connect via SSH. ### Password -Depending on your authentication method this will either be used toconnect via password to SSH, or to decrypt the SSH private key +Password of the account, or the passphrase of the private key when the authentication method is Key. - ID: `password` - Datatype: `password` @@ -116,7 +112,7 @@ Depending on your authentication method this will either be used toconnect via p ### Path -The currently selected path within your SSH instance. Auto-completion starts from user home folder, use '..' for parent directory or '/' for root directory. +Remote directory, kept with the task but not entered before the command runs. Autocompletion starts in the home directory of the account, use '..' for the parent directory or '/' for the root directory. - ID: `path` - Datatype: `string` @@ -126,7 +122,7 @@ The currently selected path within your SSH instance. Auto-completion starts fro ### Input method -Parameter to decide whether files will be used as stdin or no input is needed. If 'File input' is chosen, the input port will open for all entities withthe FileEntitySchema. +Whether incoming files are fed to the command, which also decides whether the task has an input port. - ID: `input_method` - Datatype: `string` @@ -136,7 +132,7 @@ Parameter to decide whether files will be used as stdin or no input is needed. I ### Output method -Parameter to decide which type of output the user wants. This can be either no output, a structured process output with its own schema, or a file based output +What the task emits for each run of the command, which also decides whether the task has an output port. - ID: `output_method` - Datatype: `string` @@ -146,7 +142,7 @@ Parameter to decide which type of output the user wants. This can be either no o ### Command -The command that will be executed on the SSH instance. When the inputmethod is set to 'File input', the command will be executed over these files. +Command line executed on the server. - ID: `command` - Datatype: `string` @@ -156,7 +152,7 @@ The command that will be executed on the SSH instance. When the inputmethod is s ### Timeout -A timeout for the executed command. +Seconds the task waits for the command to send data before it gives up and fails. Zero waits for as long as the command takes. - ID: `timeout` - Datatype: `Long` diff --git a/docs/build/reference/customtask/cmem_plugin_ssh-List.md b/docs/build/reference/customtask/cmem_plugin_ssh-List.md index d2bcbcf6c..e5f6fa187 100644 --- a/docs/build/reference/customtask/cmem_plugin_ssh-List.md +++ b/docs/build/reference/customtask/cmem_plugin_ssh-List.md @@ -1,6 +1,6 @@ --- title: "List SSH files" -description: "List files from a given SSH instance." +description: "List the files of a directory on an SSH server, with their metadata." icon: octicons/cross-reference-24 tags: - WorkflowTask @@ -18,43 +18,41 @@ tags: e.g. with cmemc. -This workflow task generates structured output from a specified SSH instance. +Lists the files of a directory on an SSH server, together with the metadata the +server reports for them. -By providing the hostname, username, port and authentication method, you can specify the -folder from which the data should be extracted. +The task takes no input. It emits one entity per file, holding the full remote path +in `file_name`, plus `size`, `uid`, `gid`, `mode`, `atime` and `mtime` exactly as the +server reports them: a byte count, numeric owner and group ids, numeric mode bits and +Unix timestamps. Folders are traversed but never emitted. -You can also define a regular expression to include or exclude specific files. +**Download SSH files** accepts these entities on its input port, so that listing and +downloading can be split into two steps, with transformations in between that narrow +down which of the listed files are fetched. -There is also an option to prevent files in subfolders from being included. +The **Preview results** action lists the first ten matches without running the +workflow. It stops after ten files, so it can miss the unreadable file that would make +a real run behave differently. -## Authentication Methods +## Caveats -- **Password:** Only the password will be used for authentication. The private key field is -ignored, even if filled. -- **Key:** The private key will be used for authentication. If the key is encrypted, the password -will be used to decrypt it. - -### Error handling modes - -- **Ignore:** Ignores the permission rights of files and lists them all. Skips folders when there -is no correct permission. -- **Warning:** Warns the user about files that the user has no permission rights to. Lists all files -and skips folder when there is no correct permission. -- **Error:** Throws an error when there is a single file or folder with incorrect permission rights. - -#### Note - -- If a connection cannot be established within 20 seconds, a timeout occurs. -- Currently supported key types are: RSA, ECDSA, Ed25519. -- Setting the maximum amount of workers to more than 1 may cause a Channel Exception when -the amount of files is too large +- Files the account cannot read are listed like every other file. The error handling +only decides whether they are reported on top of that, or the listing fails. +- A folder that cannot be read is skipped silently unless the error handling is set to +Error, and the files below it are then missing from the result without further notice. +- Every file is opened and one byte is read from it to find out whether it is readable, +which makes listing a large directory noticeably slower than a plain directory listing. +- The task logs in with the configured credentials only, and offers no key of the +machine it runs on. The host key of the server in turn is accepted as presented and +never checked against a known hosts list. +- Establishing the connection fails after 20 seconds. ## Parameter ### Hostname -Hostname to connect to. Usually in the form of an IP address +Host name or IP address of the SSH server. - ID: `hostname` - Datatype: `string` @@ -64,7 +62,7 @@ Hostname to connect to. Usually in the form of an IP address ### Port -The port on which the connection will be tried on. Default is 22. +TCP port the SSH server listens on. - ID: `port` - Datatype: `Long` @@ -74,7 +72,7 @@ The port on which the connection will be tried on. Default is 22. ### Username -The username with which a connection will be instantiated. +Account to log in as. - ID: `username` - Datatype: `string` @@ -84,7 +82,7 @@ The username with which a connection will be instantiated. ### Authentication method -The method that is used to connect to the SSH server. +How the task authenticates against the server. - ID: `authentication_method` - Datatype: `string` @@ -94,7 +92,7 @@ The method that is used to connect to the SSH server. ### Private key -Your private key to connect via SSH. +Private key in PEM format, used when the authentication method is Key. RSA, ECDSA and Ed25519 keys are supported. - ID: `private_key` - Datatype: `password` @@ -104,7 +102,7 @@ Your private key to connect via SSH. ### Password -Depending on your authentication method this will either be used toconnect via password to SSH, or to decrypt the SSH private key +Password of the account, or the passphrase of the private key when the authentication method is Key. - ID: `password` - Datatype: `password` @@ -114,7 +112,7 @@ Depending on your authentication method this will either be used toconnect via p ### Path -The currently selected path within your SSH instance. Auto-completion starts from user home folder, use '..' for parent directory or '/' for root directory. +Remote directory the task works in. Autocompletion starts in the home directory of the account, use '..' for the parent directory or '/' for the root directory. - ID: `path` - Datatype: `string` @@ -122,9 +120,9 @@ The currently selected path within your SSH instance. Auto-completion starts fro -### Error handling for missing permissions +### Error handling -A choice on how to handle errors concerning the permissions rights.When choosing 'ignore' all files get listed regardless if the current user has correct permission rightsWhen choosing 'warning' all files get listed however there will be a mention that some of the files are not under the users permissionsif there are anyWhen choosing 'error' the files will not get listed if therethere are files the user has no access to. +How the task reacts to files and folders the account cannot read. - ID: `error_handling` - Datatype: `string` @@ -134,7 +132,7 @@ A choice on how to handle errors concerning the permissions rights.When choosing ### No subfolder -When this flag is set, only files from the current directory will be listed. +If enabled, only the given directory is listed and its subfolders are left out. - ID: `no_subfolder` - Datatype: `boolean` @@ -144,7 +142,7 @@ When this flag is set, only files from the current directory will be listed. ### Regular expression -A regular expression used to define which files will get listed. +Regular expression a file name has to match completely to be listed. It is matched against the plain file name, not against the path. - ID: `regex` - Datatype: `string` @@ -152,9 +150,9 @@ A regular expression used to define which files will get listed. ## Advanced Parameter -### Maximum amount of workers +### Maximum number of workers -Determines the amount of workers used for concurrent thread execution of the task. Default is 1, maximum is 32. Note that too many workers can cause a ChannelException. +Number of threads that list subfolders in parallel, from 1 to 32. More threads speed up a deep folder tree, but too many parallel channels make some servers refuse them with a ChannelException. - ID: `max_workers` - Datatype: `Long` diff --git a/docs/build/reference/customtask/cmem_plugin_ssh-Upload.md b/docs/build/reference/customtask/cmem_plugin_ssh-Upload.md index 5cfc87140..79a440758 100644 --- a/docs/build/reference/customtask/cmem_plugin_ssh-Upload.md +++ b/docs/build/reference/customtask/cmem_plugin_ssh-Upload.md @@ -1,6 +1,6 @@ --- title: "Upload SSH files" -description: "Upload files to a given SSH instance." +description: "Upload files to an SSH server." icon: octicons/cross-reference-24 tags: - WorkflowTask @@ -18,29 +18,35 @@ tags: e.g. with cmemc. -This workflow task uploads files to a given SSH instance. +Uploads the files it receives to a directory on an SSH server. -By providing the hostname, username, port and authentication method, you can specify the -folder the data should be uploaded to. +Files arrive on the input port as file entities and are written into the configured +directory under their plain file name. The task has no output port: it ends the branch +of the workflow it sits in and hands nothing on to a following task. -## Authentication Methods +**Download SSH files** covers the other direction, and **Execute commands via SSH** +runs a command on the uploaded files once they are in place. -- **Password:** Only the password will be used for authentication. The private key field is -ignored, even if filled. -- **Key:** The private key will be used for authentication. If the key is encrypted, the password -will be used to decrypt it. +## Caveats -### Note - -- If a connection cannot be established within 20 seconds, a timeout occurs. -- Currently supported key types are: RSA, ECDSA, Ed25519. +- Content that is gzip compressed is unpacked before it is sent. Such a file arrives on +the server uncompressed, under an unchanged name that still ends in `.gz`. +- A file of the same name already on the server is overwritten without warning, and so +are files of the same name coming from different folders. +- The first failing upload aborts the task. Files uploaded before that stay on the +server, and nothing is rolled back. +- The target directory has to exist. It is not created. +- The task logs in with the configured credentials only, and offers no key of the +machine it runs on. The host key of the server in turn is accepted as presented and +never checked against a known hosts list. +- Establishing the connection fails after 20 seconds. ## Parameter ### Hostname -Hostname to connect to. Usually in the form of an IP address +Host name or IP address of the SSH server. - ID: `hostname` - Datatype: `string` @@ -50,7 +56,7 @@ Hostname to connect to. Usually in the form of an IP address ### Port -The port on which the connection will be tried on. Default is 22. +TCP port the SSH server listens on. - ID: `port` - Datatype: `Long` @@ -60,7 +66,7 @@ The port on which the connection will be tried on. Default is 22. ### Username -The username with which a connection will be instantiated. +Account to log in as. - ID: `username` - Datatype: `string` @@ -70,7 +76,7 @@ The username with which a connection will be instantiated. ### Authentication method -The method that is used to connect to the SSH server. +How the task authenticates against the server. - ID: `authentication_method` - Datatype: `string` @@ -80,7 +86,7 @@ The method that is used to connect to the SSH server. ### Private key -Your private key to connect via SSH. +Private key in PEM format, used when the authentication method is Key. RSA, ECDSA and Ed25519 keys are supported. - ID: `private_key` - Datatype: `password` @@ -90,7 +96,7 @@ Your private key to connect via SSH. ### Password -Depending on your authentication method this will either be used toconnect via password to SSH, or to decrypt the SSH private key +Password of the account, or the passphrase of the private key when the authentication method is Key. - ID: `password` - Datatype: `password` @@ -100,7 +106,7 @@ Depending on your authentication method this will either be used toconnect via p ### Path -The currently selected path within your SSH instance. Auto-completion starts from user home folder, use '..' for parent directory or '/' for root directory. +Remote directory the files are written to. Autocompletion starts in the home directory of the account, use '..' for the parent directory or '/' for the root directory. - ID: `path` - Datatype: `string` diff --git a/docs/build/reference/customtask/cmem_plugin_validation-validate-ValidateEntities.md b/docs/build/reference/customtask/cmem_plugin_validation-validate-ValidateEntities.md index abfb8a6ef..2f3445410 100644 --- a/docs/build/reference/customtask/cmem_plugin_validation-validate-ValidateEntities.md +++ b/docs/build/reference/customtask/cmem_plugin_validation-validate-ValidateEntities.md @@ -28,39 +28,33 @@ The used JSON Schema needs to be provided as a JSON Dataset in the project. ## Input Modes -The plugin supports two input modes for validation: - -1. **Validate Entities**: Validates entities received from the input port in the workflow. -2. **Validate JSON Dataset**: Validates a JSON dataset stored in the project. - - If the JSON dataset is a JSON array, the schema will validate each object inside the array. - - If the JSON dataset is a JSON object, it will be validated against the schema directly. - -Validated data objects can be sent to an output port for further processing in the workflow -or saved in a JSON dataset in the project. +Entities arrive on the input port by default. Switching the input mode to a JSON dataset +removes the input port, and the resources to validate are instead read from a JSON dataset in +the project. A JSON array in that dataset has each of its objects validated individually; a +JSON object is validated directly. ### Output Modes -1. **Valid JSON objects sent to Output Port**: Valid JSON objects can be sent as entities - to the output port. -2. **Saved in JSON Dataset**: Valid JSON objects can be stored in a specified JSON dataset - in the project. +Valid JSON objects are sent to an output port by default, ready for further processing in the +workflow. Switching the output mode to a JSON dataset removes the output port, and the valid +JSON objects are instead saved to a JSON dataset in the project, replacing its existing content. ### Error Handling -The task can either: +Every entity or object is validated regardless of the outcome. The task can then either: -- Fail instantly if there is a data violation, halting the workflow. -- Provide warnings in the workflow report, allowing follow-up tasks to run based on the +- Fail once validation completes if any entity has violations, halting the workflow. +- Report violations only as warnings, allowing follow-up tasks to run based on the validated data. -The error handling behavior is configurable through the `Fail on violations` parameter. +The error handling behavior is configurable through the **Fail on violations** parameter. ## Parameter ### Source / Input Mode - +Selects where entities to validate come from: the input port or a JSON dataset. - ID: `source_mode` - Datatype: `string` @@ -70,7 +64,7 @@ The error handling behavior is configurable through the `Fail on violations` par ### Target / Output Mode - +Selects where valid JSON objects are written to: the output port or a JSON dataset. - ID: `target_mode` - Datatype: `string` @@ -90,7 +84,7 @@ This dataset holds the JSON schema to use for validation. ### Fail on violations -If enabled, the task will fail on the first data violation. +If enabled, the workflow fails once validation completes if any entity has violations. All entities are validated either way; disabling this instead reports the violations as warnings. - ID: `fail_on_violations` - Datatype: `boolean` @@ -100,7 +94,7 @@ If enabled, the task will fail on the first data violation. ### Source JSON Dataset -This dataset holds the resources you want to validate. +This dataset holds the resources you want to validate. Required when Source / Input Mode is set to dataset; leave it empty when using entities. - ID: `source_dataset` - Datatype: `string` @@ -110,7 +104,7 @@ This dataset holds the resources you want to validate. ### Target JSON Dataset -This dataset will be used to store the valid JSON objects after validation. +This dataset stores the valid JSON objects after validation, replacing any existing content. Required when Target / Output Mode is set to dataset; leave it empty when using entities. - ID: `target_dataset` - Datatype: `string` diff --git a/docs/build/reference/customtask/cmem_plugin_validation-validate-ValidateGraph.md b/docs/build/reference/customtask/cmem_plugin_validation-validate-ValidateGraph.md index fa477eed8..4585ab80a 100644 --- a/docs/build/reference/customtask/cmem_plugin_validation-validate-ValidateGraph.md +++ b/docs/build/reference/customtask/cmem_plugin_validation-validate-ValidateGraph.md @@ -18,8 +18,24 @@ tags: e.g. with cmemc. -Start a graph validation process which verifies, that resources in a specific graph are valid -according to the node shapes in a shape catalog graph. +Starts a graph validation process which verifies that resources in a specific graph are valid +according to the node shapes in a shape catalog graph. The task waits for the validation to +finish before completing. + +The task has no input port. Violations found during validation are always summarized in the +workflow report, and can also be materialized into a result graph in the project. Sending each +violation as an entity to an output port is optional; when that is disabled, the task has no +output port either. + +## Error Handling + +Every resource is validated regardless of the outcome. The task can then either: + +- Fail once validation completes if any resource has violations, halting the workflow. +- Report violations only as warnings, allowing follow-up tasks to run based on the results. + +The error handling behavior is configurable through the **Fail workflow on violations** +parameter. ## Parameter @@ -56,7 +72,7 @@ In this graph, the validation results are materialized. If left empty, results a ### Clear result graph before validation - +If enabled, the existing content of the result graph is deleted before validation starts. Has no effect when Result graph is left empty. - ID: `clear_result_graph` - Datatype: `boolean` @@ -66,7 +82,7 @@ In this graph, the validation results are materialized. If left empty, results a ### Fail workflow on violations - +If enabled, the workflow fails once validation completes if any resource has violations. All resources are validated either way; disabling this instead reports the violations as warnings. - ID: `fail_on_violations` - Datatype: `boolean` @@ -76,7 +92,7 @@ In this graph, the validation results are materialized. If left empty, results a ### Output violations as entities - +If enabled, each violation is sent as an entity to an output port for further processing in the workflow. Disabling this removes the output port. - ID: `output_results` - Datatype: `boolean` @@ -86,7 +102,7 @@ In this graph, the validation results are materialized. If left empty, results a ### Resource Selection Query -The query to select the resources to validate. Use {{context_graph}} as a placeholder for the select context graph for validation. +The query to select the resources to validate. Use `{{context_graph}}` as a placeholder for the selected Context Graph. - ID: `sparql_query` - Datatype: `code-sparql` diff --git a/docs/build/reference/customtask/cmem_plugin_wfreports_get_report.md b/docs/build/reference/customtask/cmem_plugin_wfreports_get_report.md index a88f9eb2e..0f86fbdd1 100644 --- a/docs/build/reference/customtask/cmem_plugin_wfreports_get_report.md +++ b/docs/build/reference/customtask/cmem_plugin_wfreports_get_report.md @@ -18,32 +18,28 @@ tags: e.g. with cmemc. -This workflow operator retrieves a specific execution report of a workflow and outputs it -as a JSON file. +This workflow operator retrieves one execution report of a workflow and outputs it as a JSON +file. -The plugin queries the backend API to list all available reports for the given workflow and -allows you to select which report to retrieve: +## Ports -- **Latest Report**: The most recent execution report -- **Latest Report with Errors**: The most recent failed execution report -- **Latest Report with Warning or Errors**: The most recent report with warnings (successful or not) - -The report contains detailed information about the workflow execution, including task results, -execution times, and any errors or warnings. - -## Output - -The plugin outputs a single JSON file entity containing the complete workflow execution report. +The task has no input port; it queries the reports available for the selected workflow +directly. It outputs a single JSON file entity holding the report - the same JSON payload +returned for that execution, including its finish status and any warnings. ## Usage -This operator is useful for: +With no input port, this operator is typically the first task in a workflow. Its output feeds +a downstream task that reads or forwards the JSON file, for example to inspect it, archive it, +or route on the presence of errors or warnings. + +## Caveats -- Monitoring workflow execution results and failures -- Debugging recent errors or warnings -- Creating audit trails of workflow runs -- Archiving execution reports for compliance purposes -- Feeding execution data into downstream analysis tasks +- If no report matches the selected mode - none currently carries warnings or errors, or the + time period excludes every report - the output file contains an empty JSON object `{}`, not + an error or any other indication that nothing was found. +- Finding the latest report with warnings or errors is checked from newest to oldest, and may + fetch every report in scope individually before a match is found or the list is exhausted. ## Parameter @@ -70,7 +66,7 @@ Selector for reports to be shown. ### Time Period -The time period in hours of the workflow execution reports that can be listed.Allows only full hours. Defaults to 0 for every execution report. +The time period in hours of the workflow execution reports that can be listed. Allows only full hours. Defaults to 0, which lists every execution report. - ID: `time_period` - Datatype: `Long` diff --git a/docs/build/reference/customtask/cmem_plugin_yaml-parse.md b/docs/build/reference/customtask/cmem_plugin_yaml-parse.md index 04c00aa15..c68d541ae 100644 --- a/docs/build/reference/customtask/cmem_plugin_yaml-parse.md +++ b/docs/build/reference/customtask/cmem_plugin_yaml-parse.md @@ -1,6 +1,6 @@ --- title: "Parse YAML" -description: "Parses files, source code or input values as YAML documents." +description: "Parses YAML documents from a code field, entities or files and converts them to JSON." icon: octicons/cross-reference-24 tags: - WorkflowTask @@ -18,31 +18,45 @@ tags: e.g. with cmemc. -This workflow task parses YAML content from multiple sources and converts it to various output -formats. - -**Input Sources:** - -- **entities**: Parse YAML from input port entities in a workflow -- **code**: Parse YAML from directly entered source code -- **file**: Parse YAML from uploaded project file resources - -**Output Formats:** - -- **entities**: Convert parsed structure to entities for workflow processing -- **json_entities**: Output as single JSON entity to the output port -- **json_dataset**: Save parsed structure directly to a JSON dataset - -The plugin provides flexible YAML-to-JSON conversion with configurable input schema -types and paths for entity-based processing. It includes comprehensive validation and -error handling for all supported modes. +This task reads YAML documents, converts each of them to JSON, and hands the result on +either as entities or as files. + +The YAML comes from one of three places: the code field of this task, entities carrying it +as text, or files arriving on an input port. Both port modes read every entity of every +port the task declares, so a batch of files becomes a batch of results, and the number of +those ports is configurable, for documents which come from several tasks at once. +Configured to read from its code field, it declares no input port and starts the workflow. +On the output side the task either hands on the structure of the documents as entities, or +one JSON file per document. + +It usually sits between a task that produces files and one that consumes them: read a +directory of YAML, parse it here, and let a task that stores project resources keep the +JSON. Feeding the entities output into a transformation is the other common chain. + +Worth knowing before configuring it: + +- A document has to describe a mapping or a sequence. A file holding nothing but a string + or a number is rejected, since neither becomes a JSON object. +- Exactly one document is read per file. A stream of several documents separated by `---` + fails. +- Parsing is safe, so YAML tags that construct arbitrary Python objects are refused. +- Documents leaving as entities are combined into one stream whose paths are the union of + all of them, and a value a document does not carry becomes empty. Where two documents + disagree about a key - a string in one, a list in the next - the key is carried as text + in both, since one schema cannot hold it both ways. +- A key which is not a string becomes one. Mind that `on`, `yes` and `no` are booleans in + YAML, so `on:` reads back as `True` unless it is quoted in the source document. +- A returned file is named after the file it came from - after the entry, when that file + is an entry in an archive - with a `.json` suffix, made unique when two of them would + otherwise share a name. A document that did not come from a file - from the code field, + or from an entity - has no name of its own and is written as `parsed-yaml.json` instead. ## Parameter ### Source / Input Mode - +Where the YAML documents come from. - ID: `source_mode` - Datatype: `string` @@ -52,7 +66,7 @@ error handling for all supported modes. ### Target / Output Mode - +In which shape the parsed documents leave this task. - ID: `target_mode` - Datatype: `string` @@ -62,7 +76,7 @@ error handling for all supported modes. ### YAML Source Code (when using the *code* input) - +The YAML document itself, used in the 'code' source mode. - ID: `source_code` - Datatype: `code-yaml` @@ -70,29 +84,29 @@ error handling for all supported modes. ## Advanced Parameter -### YAML File (when using the *file* input) +### Number of Input Ports -Which YAML file do you want to load into a JSON dataset? The dropdown shows file resources from the current project. +How many input ports the task declares. Raise it to read from several tasks at once, in the order the ports are shown. It has no effect in the 'code' source mode, which declares no input port at all. -- ID: `source_file` -- Datatype: `string` -- Default Value: `None` +- ID: `number_of_inputs` +- Datatype: `Long` +- Default Value: `1` -### Target Dataset +### Tolerate Unusable Input -Where do you want to save the result of the conversion? The dropdown shows JSON datasets from the current project. +If enabled, a document that cannot be parsed is reported and skipped instead of stopping the task, and an input that delivers nothing at all produces an empty result. A batch in which every document fails remains an error, since something arrived and none of it could be used. -- ID: `target_dataset` -- Datatype: `string` -- Default Value: `None` +- ID: `tolerate_unusable_input` +- Datatype: `boolean` +- Default Value: `false` ### Input Schema Type / Class -In case of source mode 'entities', you can specify the requested input type. +The type the input ports ask for, used in the 'entities' source mode. Change it when the preceding task delivers entities of a different type. - ID: `input_schema_type` - Datatype: `string` @@ -102,7 +116,7 @@ In case of source mode 'entities', you can specify the requested input type. ### Input Schema Path / Property -In case of source mode 'entities', you can specify the requested input path. +The path the YAML document is read from, used in the 'entities' source mode. Change it when the preceding task carries the document in a differently named path. - ID: `input_schema_path` - Datatype: `string` diff --git a/docs/build/reference/customtask/eccencaRestOperator.md b/docs/build/reference/customtask/eccencaRestOperator.md index f44814977..0362ca116 100644 --- a/docs/build/reference/customtask/eccencaRestOperator.md +++ b/docs/build/reference/customtask/eccencaRestOperator.md @@ -1,6 +1,6 @@ --- title: "Execute REST requests" -description: "REST operator that fetches and optionally merges data from a REST endpoint. It supports executing multiple requests either via input entities that each overwrite config parameters or via paging. If you only need to download a single file, the 'Download file' operator might be the better option. Most features are currently only supported for JSON REST APIs. From multiple requests the REST operator can produce a merged JSON result, i.e. for JSON it will concatenate all results in a JSON array. Alternatively multiple results can be written directly to file (of a JSON dataset), either as a merged JSON file or one file per request inside a ZIP file. By default the output of this operator is an entity with a single property 'result', which is the (concatenated) JSON string." +description: "REST operator that fetches and optionally merges data from a REST endpoint. It supports executing multiple requests either via input entities that each overwrite config parameters or via paging. If you only need to download a single file, the 'Download file' operator might be the better option. Most features are currently only supported for JSON REST APIs. By default the output of this operator is one entity per response with a single property 'result' that holds the response body. Responses of multiple requests are merged into a single JSON array if the 'Parse JSON' operator is connected or the Accept header starts with 'application/json'. Alternatively the response/s can be written to the file of a connected file-based dataset, either as a merged JSON file (JSON dataset) or, if the dataset file ends in '.zip', one file per response inside a ZIP file." icon: octicons/cross-reference-24 tags: - WorkflowTask @@ -32,6 +32,13 @@ If the request needs authorization following parameters should be set, else ther and cannot be accessed in the user interface anymore after saving it. E.g. for OAuth the value would have the following form: `bearer `. +## Secrets in the request content and headers + +Templates of the `Content` and `HTTP headers` parameters may reference sensitive variables, e.g. `secret={{global.apiSecret}}` +as content or `X-Api-Key: {{global.apiKey}}` as a header line. The evaluated value is sent with the request but not displayed: +the template preview, the task properties and the search show the placeholder `••••••` instead. +As with password parameters, anyone who can edit the task can send the value to any host. + ## Sending multiple requests In the default configuration a single requests is sent. If multiple requests should be sent with different URLs and/or content, @@ -46,9 +53,8 @@ the configurations for these requests must be defined via the input port of the - Limit: If set to a positive number, then only that number of input entities will be processed as requests. - Offset: If set to a positive number, then that many input entities will be ignored before processing them as requests. -If the option 'Read parameters from input' is enabled, it is currently always assumed that multiple requests will be sent. -The responses must either be of type JSON, then the results are merged into a JSON array, or the 'Output result as file' -option must be enabled in order to write a merged JSON or a ZIP file. See section 'Output options' for more details. +If the option 'Read parameters from input' is enabled or a paging method is configured, it is always assumed that multiple +requests will be sent. See section 'Output options' for how the responses of multiple requests are output. ## Paging @@ -100,21 +106,49 @@ following parameter must be configured: ## Output options -By default, the response body of a request is output as value of the 'result' property of a single output entity. -If the response body needs to be processed this can e.g. be achieved with the 'Parse JSON' operator. Alternatively -the response/s can be written to a file based dataset. Currently only text based datasets are supported. +By default, the response body of a request is output as value of the 'result' property of an output entity. +If the response body needs to be processed this can e.g. be achieved with the 'Parse JSON' operator. +If multiple requests are sent, see sections 'Sending multiple requests' and 'Paging', one entity per response is output. +Only if the 'Parse JSON' operator is connected to the output or the 'Accept' parameter starts with 'application/json', +the responses are merged into a single JSON array that is output as a single entity. In that case each response of a +GET request must be valid JSON, otherwise the request is considered failed. Responses to other HTTP methods that are not +valid JSON are skipped. -The results of multiple requests, see section 'Sending multiple requests' for details, can be written to -a single, merged file (only supported for JSON) or to a ZIP archive, i.e. a file resource that must end in '.zip'. -In the latter case an entry per request is added to the ZIP file. -Currently, the following datasets support the processing of ZIP files: JSON, XML, CSV and RDF file. +Alternatively, the response/s can be written to the file of a file-based dataset: -- `Output result as file`: If enabled, instead of outputting a single entity, the result/s will be written directly +- `Output result as file`: If enabled, instead of outputting entities, the result/s will be written directly to the file of the file-based dataset that is connected to the output of this operator. -If the option 'Read parameters from input' is enabled, it is currently always assumed that multiple requests will be sent. -The responses must either be JSON, then the results are merged into a JSON array or the 'Output result as file' -option must be enabled in order to write a merged JSON or ZIP file. +With this option enabled, the operator outputs a file instead of entities. The task connected to the output must be a +file-based dataset, e.g. JSON, XML, CSV, Text or RDF file. The file of that dataset is replaced by the written file, +so its previous content is lost. Afterwards the responses can be processed like any other file of the dataset, e.g. by +a transformation that reads the dataset. Currently only text based responses are supported. Responses with HTTP status +204 (No Content) are not written. + +### Single request + +If a single request is sent, the response body is written to the dataset file as is. If the dataset file is a ZIP archive, +i.e. its name ends in '.zip', the response is written as the only entry of the archive, named as described under +'Multiple requests'. If the dataset is a JSON dataset with a file that does not end in '.zip', the response of a GET request +must be valid JSON, otherwise the request is considered failed, and the 'URL property' option, see section 'Propagating +the request URL', is applied to the valid response before it is written. + +### Multiple requests + +If multiple requests are sent, all responses are written to a single file in one of the following ways: + +- ZIP archive: If the file of the connected dataset ends in '.zip', one entry per response is added to the archive. + This applies to all file-based datasets, including a JSON dataset, for which no merged JSON array is written in that case. + The entries are named 'responseBody' followed by a ten-digit counter, e.g. `responseBody0000000001.json`. The suffix + is '.json', '.csv' or '.xml' for the JSON, CSV, Multi CSV ZIP and XML datasets and omitted for other datasets. + The following datasets can read ZIP archives: JSON, XML, CSV, Multi CSV ZIP, Text, Binary file and RDF file. An RDF file dataset + needs its 'format' parameter set, since the format is guessed from the dataset file name, which ends in '.zip'. +- Merged JSON file: If a JSON dataset with a file that does not end in '.zip' is connected, the responses are merged into + a JSON array, i.e. `[,,...]`. Each response of a GET request must be valid JSON, otherwise the + request is considered failed. For other HTTP methods, responses that are not valid JSON are skipped. The 'URL property' + option is applied to each response before it is written. + +Writing multiple responses to a dataset that is neither a JSON dataset nor based on a ZIP file fails with a validation error. ## Fine-tuning timeouts @@ -228,7 +262,7 @@ The content-type header String. This can be set in case of PUT or POST. If anoth ### Content -The content that is send with a POST, PUT or PATCH request. For handling this payload dynamically this parameter must be overwritten via the task input. +The content that is send with a POST, PUT or PATCH request. For handling this payload dynamically this parameter must be overwritten via the task input. A template of this parameter may use sensitive variables; their values are sent with the request but not displayed. - ID: `content` - Datatype: `string` @@ -238,7 +272,7 @@ The content that is send with a POST, PUT or PATCH request. For handling this pa ### HTTP headers -Configure additional HTTP headers. One header per line. Each header entry follows the curl syntax. +Configure additional HTTP headers. One header per line. Each header entry follows the curl syntax. A template of this parameter may use sensitive variables; their values are sent with the request but not displayed. - ID: `httpHeaders` - Datatype: `multiline string` @@ -378,7 +412,7 @@ The query parameter name for the next page ID that should be attached to the nex ### Output result as file -If a file based dataset is connected to the output of the REST operator, then this option can be enabled in order to overwrite the file resource of the connected dataset. This allows for handling the result of the REST request/s as a normal dataset. If a non-file based dataset is connected to this operator the execution will fail. If disabled, a single entity with a single property 'result' will be output that contains the (merged) result. +If enabled, the response/s are written to the file of the file-based dataset connected to the output, replacing its previous content. A single response is written as is, or as the only entry of a ZIP file. Multiple responses are written as one entry per response into a ZIP file if the dataset file ends in '.zip', or merged into a JSON array if a JSON dataset is connected. The task connected to the output must be a file-based dataset. If disabled, the response/s are output as entities with a single property 'result'. - ID: `outputResultAsFile` - Datatype: `boolean` diff --git a/docs/build/reference/customtask/index.md b/docs/build/reference/customtask/index.md index 5b22ea181..d52d6aa16 100644 --- a/docs/build/reference/customtask/index.md +++ b/docs/build/reference/customtask/index.md @@ -23,7 +23,7 @@ A custom workflow task is an operator that can be used in a workflow. | [Clear dataset](clearDataset.md) | Clears the dataset that is connected to the output of this operator. | | [Combine CSV files](combine-csv.md) | Combine CSV files with the same structure to one dataset. | | [Concatenate to file](ConcatenateToFile.md) | Concatenates values into a file. | - | [Create Embeddings](cmem_plugin_llm-CreateEmbeddings.md) | Fetch and output LLM created embeddings from input entities. | + | [Create Embeddings](cmem_plugin_llm-CreateEmbeddings.md) | Turn the text of each entity into a vector embedding and add it to the entity. | | [Create or Update Graph Insights Snapshots](cmem_plugin_graph_insights-Update.md) | Create a snapshot for a graph that has none, or update it if one already exists. | | [Create/Update Salesforce Objects](cmem_plugin_salesforce-workflow-operations-SobjectCreate.md) | Manipulate data in your organization's Salesforce account. | | [Delete project files](deleteProjectFiles.md) | Removes file resources from the project based on a regular expression. | @@ -33,40 +33,49 @@ A custom workflow task is an operator that can be used in a workflow. | [Download Git files](cmem_plugin_git-Download.md) | Download files from a git repository and hand them to the next task. | | [Download Nextcloud files](cmem_plugin_nextcloud-Download.md) | Download files from a given Nextcloud instance. | | [Download Office 365 Files](cmem_plugin_office365-Download.md) | Download files from Microsoft OneDrive or Sites | - | [Download SSH files](cmem_plugin_ssh-Download.md) | Download files from a given SSH instance | - | [Evaluate template](Template.md) | Evaluates a template on a sequence of entities. Can be used after a transformation or directly after datasets that output a single table, such as CSV or Excel. | + | [Download S3 objects](cmem_plugin_s3-Download.md) | Download objects from a bucket in S3 or S3-compatible object storage. | + | [Download SSH files](cmem_plugin_ssh-Download.md) | Download files from an SSH server. | + | [Evaluate template](Template.md) | Evaluates a template on a sequence of entities. Can be used after a transformation or directly after datasets that output a single table, such as CSV or Excel. A template that uses no input variables can be evaluated without an input. | | [Execute a command in a kubernetes pod](cmem_plugin_kubernetes-Execute.md) | Connect to a cluster, execute a command and gather the output. | - | [Execute commands via SSH](cmem_plugin_ssh-Execute.md) | Execute commands on a given SSH instance. | - | [Execute Instructions](cmem_plugin_llm-ExecuteInstructions.md) | Send instructions (prompt) to an LLM and process the result. | - | [Execute REST requests](eccencaRestOperator.md) | REST operator that fetches and optionally merges data from a REST endpoint. It supports executing multiple requests either via input entities that each overwrite config parameters or via paging. If you only need to download a single file, the "Download file" operator might be the better option. Most features are currently only supported for JSON REST APIs. From multiple requests the REST operator can produce a merged JSON result, i.e. for JSON it will concatenate all results in a JSON array. Alternatively multiple results can be written directly to file (of a JSON dataset), either as a merged JSON file or one file per request inside a ZIP file. By default the output of this operator is an entity with a single property 'result', which is the (concatenated) JSON string. | + | [Execute Catalog Query](cmem_plugin_querycatalog-ExecuteCatalogQuery.md) | Run a query stored in a query catalog and hand on its result. | + | [Execute commands via SSH](cmem_plugin_ssh-Execute.md) | Execute a command on an SSH server. | + | [Execute Instructions](cmem_plugin_llm-ExecuteInstructions.md) | Prompt a large language model once per entity and add the answer to the entity. | + | [Execute REST requests](eccencaRestOperator.md) | REST operator that fetches and optionally merges data from a REST endpoint. It supports executing multiple requests either via input entities that each overwrite config parameters or via paging. If you only need to download a single file, the "Download file" operator might be the better option. Most features are currently only supported for JSON REST APIs. By default the output of this operator is one entity per response with a single property 'result' that holds the response body. Responses of multiple requests are merged into a single JSON array if the 'Parse JSON' operator is connected or the Accept header starts with 'application/json'. Alternatively the response/s can be written to the file of a connected file-based dataset, either as a merged JSON file (JSON dataset) or, if the dataset file ends in '.zip', one file per response inside a ZIP file. | | [Execute Spark function](SparkFunction.md) | Applies a specified Scala function to a specified field. | | [Extract from PDF files](cmem_plugin_pdf_extract-pdf_extract-PdfExtract.md) | Extract text and tables from PDF files | | [Generate base36 IRDIs](cmem_plugin_irdi-workflow-irdi_plugin-IrdiPlugin.md) | Create unique ECLASS IRDIs. | | [Generate random values](cmem_plugin_random-GenerateEntities.md) | Generates entities with random values. | | [Generate SHACL shapes from data](cmem_plugin_shapes-plugin_shapes-ShapesPlugin.md) | Generate SHACL node and property shapes from a data graph | + | [Get Catalog Query](cmem_plugin_querycatalog-GetCatalogQuery.md) | Emit one entity describing a query stored in a query catalog. | | [Get emails](cmem-plugin-email-GetEmails.md) | Search IMAP mailbox and get email metadata or file attachments. | + | [Get ntfy attachments](cmem_plugin_ntfy-GetAttachments.md) | Download the attachments of a topic's messages from an ntfy server. | + | [Get ntfy messages](cmem_plugin_ntfy-GetMessages.md) | Read the messages of a topic from an ntfy server. | | [Get project files](getProjectFiles.md) | Get file resources from the project. | | [Get workflow report](cmem_plugin_wfreports_get_report.md) | Output a workflow execution report as a JSON file. | - | [GraphQL query](cmem_plugin_graphql-workflow-graphql-GraphQLPlugin.md) | Executes a custom GraphQL query to a GraphQL endpoint and saves result to a JSON dataset. | + | [GraphQL query](cmem_plugin_graphql-Query.md) | Sends a GraphQL query or mutation to an endpoint and returns the result as entities. | + | [Hetzner Cloud: List Resources](cmem_plugin_hetzner-ListResources.md) | Inventory all existing resources of a Hetzner Cloud project (servers, volumes, networks, load balancers, IPs, firewalls, SSH keys, snapshots, …), one entity per resource in a single union schema. | + | [Hetzner Cloud: List Server Types](cmem_plugin_hetzner-ListServerTypes.md) | List all available Hetzner Cloud server (VM) types, one entity per server type and location, including specs and per-location pricing. | | [Install Packages](cmem_plugin_packages-InstallPackages.md) | Install packages and their dependencies from a marketplace service or from package archives. | | [Join tables](Merge.md) | Joins a set of inputs into a single table. Expects a list of entity tables and links. All entity tables are joined into the first entity table using the provided links. | | [jq](cmem-plugin-jq-workflow.md) | Process a JSON document with a jq filter / program. | - | [JQL query](cmem_plugin_jira-JqlQuery.md) | Search and retrieve JIRA issues. | + | [JQL query](cmem_plugin_jira-JqlQuery.md) | Search and retrieve Jira issues. | | [JSON to File](jsonToFile.md) | Writes a JSON string held in a field on each valid incoming entity to a file. Depending on the output mode, it produces one file per entity, packs all entities into a single ZIP archive, or merges them into a single JSON array file. Produces a file entity downstream, suitable for wiring into a file-backed dataset or any operator that consumes file entities. | - | [Kafka Consumer (Receive Messages)](cmem_plugin_kafka-ReceiveMessages.md) | Reads messages from a Kafka topic and saves it to a messages dataset (Consumer). | - | [Kafka Producer (Send Messages)](cmem_plugin_kafka-SendMessages.md) | Reads a messages dataset and sends records to a Kafka topic (Producer). | + | [Kafka Consumer (Receive Messages)](cmem_plugin_kafka-ReceiveMessages.md) | Reads messages from a Kafka topic as entities, or saves them to a dataset (Consumer). | + | [Kafka Producer (Send Messages)](cmem_plugin_kafka-SendMessages.md) | Sends entities, or messages from a dataset, to a Kafka topic (Producer). | + | [List Catalog Queries](cmem_plugin_querycatalog-ListCatalogQueries.md) | Emit one entity per query stored in a query catalog. | | [List Git files](cmem_plugin_git-List.md) | List the files of a git repository, without transferring their content. | | [List Nextcloud files](cmem_plugin_nextcloud-List.md) | List directories and files from a given Nextcloud folder. | | [List Office 365 Files](cmem_plugin_office365-List.md) | List files from OneDrive or Sites | | [List project files](cmem_plugin_project_resources-List.md) | List file resources from the project. | - | [List SSH files](cmem_plugin_ssh-List.md) | List files from a given SSH instance. | + | [List S3 objects](cmem_plugin_s3-List.md) | List the objects of a bucket in S3 or S3-compatible object storage. | + | [List SSH files](cmem_plugin_ssh-List.md) | List the files of a directory on an SSH server, with their metadata. | | [Merge tables](MultiTableMerge.md) | Stores sets of instance and mapping inputs as relational tables with the mapping as an n:m relation. Expects a list of entity tables and links. All entity tables have a relation to the first entity table using the provided links. | | [Normalize units of measurement](ucumNormalizationTask.md) | Custom task that will substitute numeric values and pertaining unit symbols with a SI-system-unit normalized representation. | | [OAuth2 Authentication](cmem_plugin_auth-workflow-auth-OAuth2.md) | Provide an OAuth2 access token for other tasks (via config port). | | [Office 365 Upload Files](cmem_plugin_office365-Upload.md) | Upload files to OneDrive or a site Sharepoint | | [Parse JSON](JsonParserOperator.md) | Parses a JSON string held in a field on each incoming entity. Typically used before a transformation task. Takes exactly one input. | | [Parse XML](XmlParserOperator.md) | Takes exactly one input and reads either the defined inputPath or the first value of the first entity as XML document. Then executes the given output entity schema similar to the XML dataset to construct the result entities. | - | [Parse YAML](cmem_plugin_yaml-parse.md) | Parses files, source code or input values as YAML documents. | + | [Parse YAML](cmem_plugin_yaml-parse.md) | Parses YAML documents from a code field, entities or files and converts them to JSON. | | [Pivot](Pivot.md) | The pivot operator takes data in separate rows, aggregates it and converts it into columns. | | [Publish Packages](cmem_plugin_packages-PublishPackages.md) | Publish package archives to a marketplace service. | | [Reason](cmem_plugin_reason-plugin_reason-ReasonPlugin.md) | Performs OWL reasoning. | @@ -75,16 +84,19 @@ A custom workflow task is an operator that can be used in a workflow. | [Scheduler](Scheduler.md) | Executes a workflow at specified intervals. | | [Search addresses](SearchAddresses.md) | Looks up locations from textual descriptions using the configured geocoding API. Outputs results as RDF. | | [Search for Logs](cmem_plugin_logpoint-search_logs_task-RetrieveLogs.md) | Search and retrieve logs from a Logpoint SIEM system with flexible schema output. | - | [Search Vector Embeddings](cmem_plugin_pgvector-Search.md) | Search for top-k metadata stored in Postgres Vector Store (PGVector). | + | [Search LDAP directory](cmem_plugin_ldap-Search.md) | Query LDAP directory services such as OpenLDAP or Active Directory. | + | [Search Vector Embeddings](cmem_plugin_pgvector-Search.md) | Search for the top-k nearest matches to a query embedding in Postgres Vector Store (PGVector). | | [Select random entities](cmem_plugin_random-SelectEntities.md) | Select X random entities from an input dataset. | | [Send email](SendEMail.md) | Sends an email using an SMTP server. | - | [Send Mattermost messages](cmem_plugin_mattermost.md) | Send messages to Mattermost channels and/or users. | + | [Send Mattermost messages](cmem_plugin_mattermost.md) | Post messages to Mattermost users and channels through a bot account. | + | [Send ntfy attachment](cmem_plugin_ntfy-SendAttachment.md) | Send a file as an attachment to a topic on an ntfy server. | + | [Send ntfy message](cmem_plugin_ntfy-SendMessage.md) | Send a push notification to a topic on an ntfy server. | | [Set execution variable](setExecutionVariableOperator.md) | Sets an execution variable to the first value of the (single) input and passes the input through unchanged. The variable is written to the 'execution' scope and can be read downstream as 'execution.'. Only works while running inside a workflow execution. | | [Set or Overwrite parameters](cmem_plugin_parameters-ParametersPlugin.md) | Connect this task to a config port of another task in order to set or overwrite the parameter values of this task. | | [Set parameters](setParameters.md) | Set and overwrite parameters of a task. | | [SHACL validation with pySHACL](shacl-pyshacl.md) | Performs SHACL validation with pySHACL. | | [SOQL query (Salesforce)](cmem_plugin_salesforce-SoqlQuery.md) | Executes a custom Salesforce Object Query (SOQL) to return sets of data your organization's Salesforce account. | - | [Spark SQL query](CustomSQLExecution.md) | Executes a custom SQL query on the first input Spark dataframe and returns the result as its output. | + | [Spark SQL query](CustomSQLExecution.md) | Executes a Spark SQL query on the input entities and returns the result as its output. | | [SPARQL Construct query](sparqlCopyOperator.md) | A task that executes a SPARQL Construct query on a SPARQL enabled data source and outputs the SPARQL result. If the result should be written to the same RDF store it is read from, the SPARQL Update operator is preferable. | | [SPARQL Select query](sparqlSelectOperator.md) | A task that executes a SPARQL Select query and outputs the SPARQL result. | | [SPARQL Update query](sparqlUpdateOperator.md) | A task that outputs SPARQL Update queries for every entity from the input based on a SPARQL Update template. The output of this operator should be connected to the SPARQL datasets to which the results should be written. | @@ -97,9 +109,10 @@ A custom workflow task is an operator that can be used in a workflow. | [Upload File to Knowledge Graph](eccencaDataPlatformGraphStoreFileUploadOperator.md) | Uploads an N-Triples or Turtle (limited support) file from the file repository to a 'Knowledge Graph' dataset. The output of this operatorcan be the input of datasets that support graph store file upload, e.g. 'Knowledge Graph'. The file will be uploaded to the graph specified in that dataset. | | [Upload files to CKAN](cmem_plugin_ckan-Upload.md) | Publish files as the files of a dataset on a CKAN service. | | [Upload files to Nextcloud](cmem_plugin_nextcloud-Upload.md) | Upload files to a given Nextcloud instance. | + | [Upload files to S3](cmem_plugin_s3-Upload.md) | Upload files to a bucket in S3 or S3-compatible object storage. | | [Upload Git files](cmem_plugin_git-Upload.md) | Commit the files of the preceding task into a git repository. | | [Upload local files](cmem_plugin_project_resources-UploadLocalFiles.md) | Replace a file dataset resource with a local file or upload multiple local files to a project. | - | [Upload SSH files](cmem_plugin_ssh-Upload.md) | Upload files to a given SSH instance. | + | [Upload SSH files](cmem_plugin_ssh-Upload.md) | Upload files to an SSH server. | | [Validate Entities](cmem_plugin_validation-validate-ValidateEntities.md) | Use a JSON schema to validate entities or a JSON dataset. | | [Validate Knowledge Graph](cmem_plugin_validation-validate-ValidateGraph.md) | Use SHACL shapes to validate resources in a Knowledge Graph. | | [Validate OWL consistency](cmem_plugin_reason-plugin_validate-ValidatePlugin.md) | Validates the consistency of an OWL ontology. | diff --git a/docs/build/reference/customtask/sparqlCopyOperator.md b/docs/build/reference/customtask/sparqlCopyOperator.md index 12a255b89..7c1badd8f 100644 --- a/docs/build/reference/customtask/sparqlCopyOperator.md +++ b/docs/build/reference/customtask/sparqlCopyOperator.md @@ -12,7 +12,50 @@ tags: -A task that executes a SPARQL Construct query on a SPARQL enabled data source and outputs the SPARQL result. If the result should be written to the same RDF store it is read from, the SPARQL Update operator is preferable. +The SPARQL Construct query operator runs a [SPARQL 1.1 CONSTRUCT query](https://www.w3.org/TR/sparql11-query/#construct) +against a connected RDF dataset and passes the resulting RDF statements to the next workflow task. +Connect an RDF dataset as its input and an RDF output dataset as its output to write the result. +The operator itself does not write to a dataset. + +## Construct query + + +The **Construct query** parameter defines which RDF statements to produce. For example, this query copies names +from the source dataset into a different property in the output: + +```sparql +PREFIX ex: + +CONSTRUCT { ?person ex:displayName ?name } +WHERE { ?person ex:name ?name } +``` + +The query must be a valid CONSTRUCT query; other query forms such as SELECT and UPDATE are rejected when the task is +configured. The query is executed as written, without template variable substitution. Include any required `PREFIX` +declarations in the query. + +The source dataset's **Graph** setting is not automatically added to this CONSTRUCT query. To read a particular named +graph, specify it in the query, for example: + +```sparql +CONSTRUCT { ?s ?p ?o } +WHERE { GRAPH { ?s ?p ?o } } +``` + +Configure the destination graph on the connected output dataset if the result should be written to a named graph. + +## Use temporary file + + +**Use temporary file** is enabled by default. The operator saves the complete CONSTRUCT result to a temporary file +before passing it to the output. This avoids reading from a store while the downstream task writes the result back +to that same store. It requires enough temporary disk space for the result; the file is removed after execution. + +Disable this option to pass the result to the output without first saving it to a temporary file. This avoids the +intermediate file, but reading and writing may then overlap. The option controls this operator's intermediate file, +not how the source endpoint processes the query internally. + +If the intention is to change data within the same RDF store, consider the **SPARQL Update query** operator instead. ## Parameter @@ -29,7 +72,7 @@ A SPARQL 1.1 construct query ### Use temporary file -When copying directly to the same SPARQL Endpoint or when copying large amounts of triples, set to True by default +Save the complete CONSTRUCT result to a temporary file before passing it to the output. Useful when reading from and writing to the same store. Enabled by default. - ID: `tempFile` - Datatype: `boolean` diff --git a/docs/build/reference/customtask/sparqlSelectOperator.md b/docs/build/reference/customtask/sparqlSelectOperator.md index b2bb44891..a03a0ca92 100644 --- a/docs/build/reference/customtask/sparqlSelectOperator.md +++ b/docs/build/reference/customtask/sparqlSelectOperator.md @@ -122,16 +122,22 @@ will still cause `property` to appear in the inferred input schema. ### Output schema inference -The output schema is derived from the raw template by a heuristic, without rendering it. The heuristic takes -the projection between `SELECT` and the first `WHERE`, `FROM` or `{`, drops a leading `DISTINCT` / `REDUCED`, -and then: - -- For `SELECT *`, collects every distinct `?var` token in the query. +The output schema is derived from the raw template by a heuristic, without rendering it. The heuristic first +blanks out line comments, Jinja comments, string literals, IRIs and Jinja tags, so that nothing inside them is +mistaken for query structure. It then takes the projection between `SELECT` and the first `WHERE`, `FROM` or `{` +outside of parentheses (a keyword inside a name, e.g. `?from` or `ex:where`, is not a boundary), drops a leading +`DISTINCT` / `REDUCED`, and then: + +- For `SELECT *`, collects every distinct `?var` token in the rest of the query, unless a sub-select, `EXISTS` or `MINUS` + could bind variables that are not part of the result. - Otherwise, collects each top-level `?var` and the trailing `AS ?alias` from parenthesised expressions (e.g. `(COUNT(?s) AS ?count)` yields `count`). -Each variable becomes a string-typed path. If no variables can be detected (e.g. the projection is produced by -a Jinja expression), the output port is reported with an unknown schema. +Each variable becomes a string-typed path. If no variables can be detected, or a Jinja tag could change the set +of result variables (any block tag before the query, a tag inside the projection, or, for `SELECT *`, a tag anywhere after +the projection that is not inside an IRI or string literal), the output port is reported with an unknown schema and the +schema is taken from the actual query results at execution time. The heuristic need not be complete but must not be +wrong, so whenever it is in doubt (for example on unbalanced parentheses) it reports an unknown schema as well. ### Validation diff --git a/docs/build/reference/dataset/Jdbc.md b/docs/build/reference/dataset/Jdbc.md index 2beef8065..a7af00436 100644 --- a/docs/build/reference/dataset/Jdbc.md +++ b/docs/build/reference/dataset/Jdbc.md @@ -224,7 +224,7 @@ The strategy decides how the source system is queried. ### Write strategy -If this dataset is written to, it can be selected if data is overwritten or appended.' +How this dataset is written to. 'Create table' fails if the table already exists, 'Overwrite table' deletes an existing table before writing, and 'Append rows' adds records to an existing table, whose columns need to match the mapping. Creating and overwriting tables always needs the Spark modules. Appending rows needs them too, unless the database supports importing CSV files. - ID: `writeStrategy` - Datatype: `enumeration` diff --git a/docs/build/reference/dataset/SnowflakeJdbc.md b/docs/build/reference/dataset/SnowflakeJdbc.md index ab428dadb..f4281e4e9 100644 --- a/docs/build/reference/dataset/SnowflakeJdbc.md +++ b/docs/build/reference/dataset/SnowflakeJdbc.md @@ -230,7 +230,7 @@ Parameters related to writing to the database. #### Write strategy -If this dataset is written to, it can be selected if data is overwritten or appended. +How this dataset is written to. 'Create table' fails if the table already exists, 'Overwrite table' deletes an existing table before writing, and 'Append rows' adds records to an existing table, whose columns need to match the mapping. Creating and overwriting tables always needs the Spark modules. Appending rows needs them too, unless the database supports importing CSV files. - ID: `write.writeStrategy` - Datatype: `enumeration` diff --git a/docs/build/reference/dataset/file.md b/docs/build/reference/dataset/file.md index 4ec65e9dd..3eee237b0 100644 --- a/docs/build/reference/dataset/file.md +++ b/docs/build/reference/dataset/file.md @@ -18,7 +18,7 @@ The **RDF file** dataset plugin reads RDF data from a local file (or ZIP archive Typical use cases: -- Loading small to medium RDF files (e.g. `.ttl`, `.nt`, `.rdf`, `.nq`) into a project. +- Loading small to medium RDF files (e.g. `.ttl`, `.nt`, `.rdf`, `.nq`, `.jsonld`) into a project. - Providing a local RDF snapshot for exploration, mapping, or linking. - Exporting RDF data from a workflow into a simple N-Triples file. @@ -57,12 +57,13 @@ The detailed parameter list is generated in the UI. This section explains how to ### 3.2 Format -- Optional parameter that specifies the RDF serialization format (e.g. `RDF/XML`, `N-Triples`, `N-Quads`, `Turtle`). +- Optional parameter that specifies the RDF serialization format (e.g. `RDF/XML`, `N-Triples`, `N-Quads`, `Turtle`, `JSON-LD`). - If left empty: - the format is **auto-detected** from the file name/extension, - if no format can be detected, an error is raised and the dataset will not load. - Reading: - - Supported formats include at least `RDF/XML`, `N-Triples`, `N-Quads`, and `Turtle`. + - Supported formats include at least `RDF/XML`, `N-Triples`, `N-Quads`, `Turtle`, `RDF/JSON`, and `JSON-LD`. + - `JSON-LD` files (`.jsonld`) are read as JSON-LD 1.1. A context that the file references by URL instead of embedding it is fetched when the file is loaded, so that URL must be reachable from the server. - Writing: - **Only `N-Triples` is supported for writing.** - For other formats, the dataset is effectively read-only. @@ -71,7 +72,7 @@ The detailed parameter list is generated in the UI. This section explains how to - Optional graph name (IRI) that tells the dataset which graph to expose. - Behaviour: - - For **graph-aware formats** (e.g. `N-Quads`): + - For **graph-aware formats** (e.g. `N-Quads`, or `JSON-LD` files that contain named graphs): - you must provide a graph name to select which graph is used. - For **graph-less formats** (e.g. `N-Triples`, `Turtle`): - the graph parameter is ignored and the data is treated as a single default graph. diff --git a/docs/build/reference/dataset/json.md b/docs/build/reference/dataset/json.md index 47e30695b..e8bd17b34 100644 --- a/docs/build/reference/dataset/json.md +++ b/docs/build/reference/dataset/json.md @@ -24,6 +24,8 @@ For reading, the JSON dataset supports a number of special paths: - `#text` retrieves the text of the selected node. - The backslash can be used to navigate to the parent JSON node, e.g., `\parent/key`. The name of the backslash key (here `parent`) is ignored. +Keys that are an empty string (`"": ...`) are ignored silently, without a warning or error: they are not offered as paths and cannot be addressed by a path. Their values are still included in `#text` and matched by `*`. + ## Writing When writing JSON, all entities need to possess a unique URI. Writing multiple root entities with the same URI will result in multiple entries in the generated JSON. If multiple nested entities with the same URI are written, only the last entity with a given URI will be written. diff --git a/docs/build/reference/dataset/neo4j.md b/docs/build/reference/dataset/neo4j.md index d00f5800b..d16a3faee 100644 --- a/docs/build/reference/dataset/neo4j.md +++ b/docs/build/reference/dataset/neo4j.md @@ -33,13 +33,17 @@ Neo4j _labels_ are comparable with _classes_ in RDF (not to be confused with lab When writing entities to the Neo4j dataset, the following _labels_ will be added to each generated node: -- For each entity _type_ (such as the _type_ set in a mapping), a _label_ will be added to the node in Neo4j. +- For each `rdf:type` value of the entity, a _label_ will be added to the node in Neo4j. + This covers all target entity types set in a mapping as well as mapping rules with the target property `rdf:type`, which allow to set the labels per entity, for instance from the classes of the source resources. Since _types_ in eccenca DataIntegration are usually URIs, they will be converted according to the rules further down. - The _label_ as configured by the _label_ parameter on the Neo4j dataset itself. This is typically used to identify all entities that have been written by a certain Neo4j dataset specification in the project. For instance, if two Neo4j dataset specifications are added to a project - both writing to the same Neo4j database - different labels can be set to distinguish both sets of entities. In that respect it may be used to model a similar concept as _graphs_ in RDF. +Labels are only added, never removed. +If the types of an entity change between two runs, clear the dataset before writing, otherwise the node keeps the labels of the previous run as well. + ### Relationships A relationship connects two nodes in Neo4j. diff --git a/docs/build/reference/transformer/Combine/concat.md b/docs/build/reference/transformer/Combine/concat.md index 79c4a3606..630209f50 100644 --- a/docs/build/reference/transformer/Combine/concat.md +++ b/docs/build/reference/transformer/Combine/concat.md @@ -172,3 +172,4 @@ Handle missing values as empty strings. * [concatPairwise](concatPairwise.md) — Concatenate takes the Cartesian product of all inputs and produces one string per combination. Concatenate pairwise aligns values by position and produces one string per position, truncating to the shortest input. * [concatMultiValues](concatMultiValues.md) — Passing multiple values to a single input of Concatenate does not combine them — it multiplies the output. Concatenate multiple values is the plugin that collapses multiple values within an input into one string, producing exactly one result per input. +* [inputHash](../Value/inputHash.md) — Concatenate returns one string only when each input provides exactly one value; an input with more than one value instead returns the Cartesian product as separate results. Combined input hash always returns exactly one hash value, no matter how many values or ports are connected. diff --git a/docs/build/reference/transformer/Combine/concatMultiValues.md b/docs/build/reference/transformer/Combine/concatMultiValues.md index eb43f5f55..14fa9153d 100644 --- a/docs/build/reference/transformer/Combine/concatMultiValues.md +++ b/docs/build/reference/transformer/Combine/concatMultiValues.md @@ -152,3 +152,4 @@ No description ## Related Plugins * [concat](concat.md) — Concatenate multiple values collapses all values within each input into one string, preserving the boundary between inputs. Concatenate crosses that boundary — it takes one value from each input and produces all combinations, so the output grows with the number of inputs and values. +* [inputHash](../Value/inputHash.md) — Combined input hash always produces exactly one hash value, however many ports or values are connected. Concatenate multiple values instead combines everything on one port into a single string, so connecting more ports returns more separate strings instead of one. diff --git a/docs/build/reference/transformer/Combine/zip.md b/docs/build/reference/transformer/Combine/zip.md index d2f63d340..9f00f8fc8 100644 --- a/docs/build/reference/transformer/Combine/zip.md +++ b/docs/build/reference/transformer/Combine/zip.md @@ -112,3 +112,4 @@ Separator to be inserted between two concatenated strings. The text can contain ## Related Plugins * [concatPairwise](concatPairwise.md) — Zip handles unequal input lengths by padding, not truncating, and is constrained to exactly two inputs. Concatenate pairwise removes that constraint — it accepts any number of inputs — but resolves the length mismatch by stopping at the shortest. +* [inputHash](../Value/inputHash.md) — Zip pairs exactly two inputs position by position and returns one output per pair, using the same optional separator between values. Combined input hash instead combines every connected value into a single hash, regardless of how many ports are used. diff --git a/docs/build/reference/transformer/Conditional/ifMatchesRegex.md b/docs/build/reference/transformer/Conditional/ifMatchesRegex.md index 5fa7b9b98..2a05c6c95 100644 --- a/docs/build/reference/transformer/Conditional/ifMatchesRegex.md +++ b/docs/build/reference/transformer/Conditional/ifMatchesRegex.md @@ -139,3 +139,4 @@ No description * [validateRegex](../Validation/validateRegex.md) — A regular expression match drives both operators, but the outcome differs. The “If matches regex” plugin routes between alternative input values, while the “Validate regex” plugin draws an acceptance boundary by deciding whether the checked value is valid. * [regexSelect](../Selection/regexSelect.md) — The Regex selection plugin marks match positions by emitting copies of a provided output value wherever a regular expression matches the checked value sequence. The If matches regex plugin uses a regular expression match as a branch decision between alternative input values rather than producing positional markers. * [regexExtract](../Extract/regexExtract.md) — The Regex extract plugin returns the matching content from the input string, or the first capturing group if the regular expression contains capturing groups. The If matches regex plugin does not return matched content; it uses the match only to choose which of the provided input values becomes the output. +* [filterByRegex](../Filter/filterByRegex.md) — Filter by regex looks at values one at a time, deciding for each whether it belongs in the output. If matches regex takes an entirely different approach: the whole first sequence is checked at once, and whichever of the two other sequences applies comes back exactly as given. diff --git a/docs/build/reference/transformer/Extract/regexExtract.md b/docs/build/reference/transformer/Extract/regexExtract.md index cce3e7b29..3f4cd192b 100644 --- a/docs/build/reference/transformer/Extract/regexExtract.md +++ b/docs/build/reference/transformer/Extract/regexExtract.md @@ -178,3 +178,4 @@ If true, all matches are extracted. If false, only the first match is extracted * [regexSelect](../Selection/regexSelect.md) — The Regex selection plugin does not return matched text at all. It emits copies of a provided output value at the positions where the checked values match the regular expressions, while the Regex extract plugin returns the matched substring or capturing-group content. * [ifMatchesRegex](../Conditional/ifMatchesRegex.md) — The If matches regex plugin uses the match only as a decision about which provided input value to return. The Regex extract plugin uses the match as the produced content, so the output is derived from the matched region rather than from branch inputs. * [validateRegex](../Validation/validateRegex.md) — The Validate regex plugin keeps the original value only when the full value matches the configured regular expression and otherwise fails validation. The Regex extract plugin returns match-derived output and can return an empty result when nothing matches. +* [filterByRegex](../Filter/filterByRegex.md) — Filter by regex passes a value through exactly as it was, or excludes it — no partial output exists. Regex extract never returns the value itself; its output is always a fragment of the match, a substring or a capturing group. diff --git a/docs/build/reference/transformer/Filter/filterByLength.md b/docs/build/reference/transformer/Filter/filterByLength.md index c10844a1e..30029df3e 100644 --- a/docs/build/reference/transformer/Filter/filterByLength.md +++ b/docs/build/reference/transformer/Filter/filterByLength.md @@ -38,3 +38,7 @@ No description ## Advanced Parameter `None` + +## Related Plugins + +- [filterByRegex](filterByRegex.md) — Filter by regex tests each value against a pattern; the length is never a factor. Filter by length checks the length bounds only; a pattern is never a factor. diff --git a/docs/build/reference/transformer/Filter/filterByRegex.md b/docs/build/reference/transformer/Filter/filterByRegex.md index ac817a7cd..fbab17c02 100644 --- a/docs/build/reference/transformer/Filter/filterByRegex.md +++ b/docs/build/reference/transformer/Filter/filterByRegex.md @@ -1,6 +1,6 @@ --- title: "Filter by regex" -description: "Removes all strings that do NOT match a regex. If 'negate' is true, only strings will be removed that match the regex." +description: "This transformer filters values by a regular expression: values matching the regex are kept, unless 'negate' is set, which keeps only the values that don't. By default the regex must match a value in full; setting 'contains' relaxes that so any value containing a match is kept instead." icon: octicons/cross-reference-24 tags: - TransformOperator @@ -12,28 +12,226 @@ tags: -Removes all strings that do NOT match a regex. If 'negate' is true, only strings will be removed that match the regex. +## Description + +The Filter by regex plugin keeps or drops values based on a regular expression. By default, a value is kept only if it matches the regex in full; setting `contains` relaxes that to keep any value the regex matches anywhere within. Setting `negate` inverts whichever of those two decisions is in effect. + +## Notes on regular expressions + +Attention: regex metacharacters in the pattern have to be escaped to be matched literally, e.g. a literal dot needs `\.`, not `.`, which otherwise means "any character." + +A pattern that can match an empty string, such as `a*`, matches every value under `contains`, since an empty match always "occurs somewhere" in any string. + +A regex that fails to compile throws a `PatternSyntaxException` rather than silently doing nothing. + +Attention: This happens during _execution_. +In other words: It's technically possible to save _invalid_ regular expressions. + +### Note for advanced users + +A compilation of the available constructs for building regular expressions is available in the +[API of the Java `Pattern`](https://docs.oracle.com/en/java/javase/25/docs/api/java.base/java/util/regex/Pattern.html). + +## Missing or empty input + +If no input is connected to this transformer, it throws a `ValidationException`. If an input is connected but carries no values, the transformer returns nothing for that entity — this is not an error. + +## Examples + +**Notation:** List of values are represented via square brackets. Example: `[first, second]` represents a list of two values "first" and "second". + +--- +**Keeps only values that fully match the regex:** + +* Parameters + * regex: `abc` + +* Input values: + 1. `[abc, xabcx, xyz]` + +* Returns: `[abc]` + + +--- +**Keeps only values consisting of exactly three word characters:** + +* Parameters + * regex: `\w\w\w` + +* Input values: + 1. `[abc, ab, abcd, a-x]` + +* Returns: `[abc]` + + +--- +**Excludes values that fully match the regex, keeping the rest:** + +* Parameters + * regex: `abc` + * negate: `true` + +* Input values: + 1. `[abc, xabcx, xyz]` + +* Returns: `[xabcx, xyz]` + + +--- +**Keeps values that contain a match anywhere, not just full matches:** + +* Parameters + * regex: `abc` + * contains: `true` + +* Input values: + 1. `[abc, xabcx, xyz]` + +* Returns: `[abc, xabcx]` + + +--- +**Excludes values that contain a match anywhere, keeping the rest:** + +* Parameters + * regex: `abc` + * negate: `true` + * contains: `true` + +* Input values: + 1. `[abc, xabcx, xyz]` + +* Returns: `[xyz]` + + +--- +**A pattern matching zero characters finds a match in any value, so 'contains' keeps everything:** + +* Parameters + * regex: `a*` + * contains: `true` + +* Input values: + 1. `[xyz, aaa, ]` + +* Returns: `[xyz, aaa, ]` + + +--- +**An anchored pattern still only matches at the anchored position, even with 'contains' set:** + +* Parameters + * regex: `^abc` + * contains: `true` + +* Input values: + 1. `[abcxyz, xabcx, xyz]` + +* Returns: `[abcxyz]` + + +--- +**Throws when the regex fails to compile:** + +* Parameters + * regex: `(` + +* Input values: + 1. `[abc]` + +* Returns: `[]` +* **Throws error:** `PatternSyntaxException` + + +--- +**A backslash-escaped metacharacter in the regex is matched literally:** + +* Parameters + * regex: `a\.b` + +* Input values: + 1. `[a.b, aXb]` + +* Returns: `[a.b]` + + +--- +**A whitespace-only value matches a whitespace pattern:** + +* Parameters + * regex: `\s+` + +* Input values: + 1. `[ , abc]` + +* Returns: `[ ]` + + +--- +**A Unicode value matches a literal pattern correctly:** + +* Parameters + * regex: `café` + +* Input values: + 1. `[café, cafe]` + +* Returns: `[café]` + + +--- +**Returns nothing when the connected input carries no values:** + +* Parameters + * regex: `abc` + +* Input values: + 1. `[]` + +* Returns: `[]` + + +--- +**Throws when no input is connected:** + +* Parameters + * regex: `abc` + +* Returns: `[]` +* **Throws error:** `ValidationException` + + ## Parameter ### Regex -No description +The regular expression to test each value against. -- ID: `regex` -- Datatype: `string` -- Default Value: `None` +* ID: `regex` +* Datatype: `string` +* Default Value: `None` ### Negate -No description +If true, keeps values that do not match instead of values that do. + +* ID: `negate` +* Datatype: `boolean` +* Default Value: `false` + + + +### Contains + +If true, the pattern only needs to occur somewhere in the value. -- ID: `negate` -- Datatype: `boolean` -- Default Value: `false` +* ID: `contains` +* Datatype: `boolean` +* Default Value: `false` ## Advanced Parameter @@ -41,4 +239,10 @@ No description ## Related Plugins -- [regexSelect](../Selection/regexSelect.md) — Filter by regex keeps or drops values from the input sequence based on full-string matching. Regex selection keeps the checked value out of the output and instead returns a pattern-list-shaped result filled with the provided output value where a pattern matches. +* [regexSelect](../Selection/regexSelect.md) — Filter by regex keeps or drops values from the input sequence based on regex matching. Regex selection keeps the checked value out of the output and instead returns a pattern-list-shaped result filled with the provided output value where a pattern matches. +* [validateRegex](../Validation/validateRegex.md) — Validate regex throws on the first non-matching value, failing the whole operation. Filter by regex just drops it and keeps the rest. +* [regexReplace](../Replace/regexReplace.md) — Filter by regex is a structural filter: it never changes a value's content, only whether that value appears in the output. Regex replace works the other way: it changes what's inside each value, but every value that goes in still comes out. +* [regexExtract](../Extract/regexExtract.md) — Regex extract never returns the original value — only part of the match itself, a substring or a capturing group. By contrast, Filter by regex returns only the original value, unmodified, or nothing at all. +* [ifMatchesRegex](../Conditional/ifMatchesRegex.md) — If matches regex checks one sequence for a match, then returns one of two candidate sequences, completely unchanged. Each value in a single sequence is tested on its own under Filter by regex, kept or dropped to build the output. +* [filterByLength](filterByLength.md) — Filter by length only checks length, bounded by a minimum and a maximum. Length is irrelevant to Filter by regex; only the pattern matters. +* [removeValues](removeValues.md) — Remove values checks each value against a fixed, comma-separated blacklist — an exact, case-insensitive match, nothing computed. Filter by regex checks each value against a regular expression instead, which can be a single word or a broad, open-ended pattern. diff --git a/docs/build/reference/transformer/Filter/removeValues.md b/docs/build/reference/transformer/Filter/removeValues.md index 615a983f4..391216bea 100644 --- a/docs/build/reference/transformer/Filter/removeValues.md +++ b/docs/build/reference/transformer/Filter/removeValues.md @@ -33,3 +33,4 @@ No description - [removeEmptyValues](removeEmptyValues.md) — Remove values works from a blacklist: any value matching a word in that list is dropped. Remove empty values has no such list; it removes only empty strings. - [removeDuplicates](../Normalize/removeDuplicates.md) — The two plugins filter on different grounds. Remove values drops a value because of what it is; Remove duplicates drops a value because it already appeared earlier in the sequence. +- [filterByRegex](filterByRegex.md) — Every value Remove values can exclude has to appear in the blacklist by name. Filter by regex can exclude many values through one pattern, without naming any of them. diff --git a/docs/build/reference/transformer/Replace/regexReplace.md b/docs/build/reference/transformer/Replace/regexReplace.md index 5b95c2618..7bcacb278 100644 --- a/docs/build/reference/transformer/Replace/regexReplace.md +++ b/docs/build/reference/transformer/Replace/regexReplace.md @@ -148,3 +148,4 @@ The replacement of each match * [regexSelect](../Selection/regexSelect.md) — The Regex replace plugin rewrites a string by substituting every match with the configured replacement, so the output stays a transformed version of the input text. The Regex selection plugin turns matching into positional markers by emitting a provided output value at the regex positions that match the checked value. * [validateRegex](../Validation/validateRegex.md) — The Regex replace plugin rewrites a string by substituting every regex match and returns the rewritten value. The Validate regex plugin keeps the value only when the full value matches the regex and otherwise fails validation, so it serves as a format check before or after rewriting. * [replace](replace.md) — The Replace plugin swaps one fixed substring for another everywhere it occurs. The Regex replace plugin does the same global substitution, but what counts as a hit is described by a regular expression. +* [filterByRegex](../Filter/filterByRegex.md) — Filter by regex decides which values make it into the output, but never alters what a kept value contains. Regex replace keeps every value without exception; what changes is the content itself. diff --git a/docs/build/reference/transformer/Selection/regexSelect.md b/docs/build/reference/transformer/Selection/regexSelect.md index 88e2e40cd..843019b20 100644 --- a/docs/build/reference/transformer/Selection/regexSelect.md +++ b/docs/build/reference/transformer/Selection/regexSelect.md @@ -98,4 +98,4 @@ No description * [regexExtract](../Extract/regexExtract.md) — The Regex selection plugin returns the provided output value in a result sequence aligned with the regex list, filling only the positions whose pattern matches the checked value. The Regex extract plugin returns the matched substring itself, or the first capturing group, so the output is taken from the input text rather than from the provided output value. * [ifMatchesRegex](../Conditional/ifMatchesRegex.md) — The If matches regex plugin returns one of the provided branch inputs based on whether the checked value matches. The Regex selection plugin returns a result sequence aligned with the pattern list, placing the provided output value at every matching position. * [regexReplace](../Replace/regexReplace.md) — The Regex replace plugin returns a rewritten string by replacing every match inside the input text with the configured replacement. The Regex selection plugin returns a result sequence aligned with the pattern list and fills each matching position with the provided output value. -* [filterByRegex](../Filter/filterByRegex.md) — The Regex selection plugin keeps the checked value out of the output and instead returns a pattern-list-shaped result filled with the provided output value where a pattern matches. The Filter by regex plugin keeps or drops values from the input sequence based on full-string matching. +* [filterByRegex](../Filter/filterByRegex.md) — The Regex selection plugin keeps the checked value out of the output and instead returns a pattern-list-shaped result filled with the provided output value where a pattern matches. The Filter by regex plugin keeps or drops values from the input sequence based on regex matching. diff --git a/docs/build/reference/transformer/Sequence/getValueByIndex.md b/docs/build/reference/transformer/Sequence/getValueByIndex.md index 6973fc9f8..32dd7727a 100644 --- a/docs/build/reference/transformer/Sequence/getValueByIndex.md +++ b/docs/build/reference/transformer/Sequence/getValueByIndex.md @@ -1,6 +1,6 @@ --- title: "Get value by index" -description: "Returns the value found at the specified index. Fails or returns an empty result depending on failIfNoFound is set or not. Please be aware that this will work only if the data source supports some kind of ordering like XML or JSON. This is probably not a good idea to do with RDF models. If emptyStringToEmptyResult is true then instead of a result with an empty String, an empty result is returned." +description: "Returns the value found at the specified index. A negative index counts from the end of the sequence." icon: octicons/cross-reference-24 tags: - TransformOperator @@ -12,11 +12,93 @@ tags: -Returns the value found at the specified index. Fails or returns an empty result depending on failIfNoFound is set or not. - Please be aware that this will work only if the data source supports some kind of ordering like XML or JSON. This - is probably not a good idea to do with RDF models. +## Description + +This transformer extracts a single value from a sequence at a fixed position — useful whenever a data source +returns its values in a known, meaningful order and only one of them is needed. A positive index counts from the +start of the sequence, `0`-based. A negative index counts from the end — `-1` is the last value, `-2` the +second-to-last, and so on. An index equal to the negative length of the sequence wraps around to the first value, +rather than falling out of range. + +If no value exists at the given index, the behavior depends on `failIfNotFound`: by default the input is simply +dropped from the result, but if `failIfNotFound` is enabled, the transformation fails instead. + +Each input value is a sequence in its own right, and the index is resolved independently for each one. If this +transformer receives several input values, every one of them is indexed on its own, not treated as a single +combined sequence. + +Indexing only makes sense where the data source preserves some kind of order, such as XML or JSON. RDF has no +inherent ordering, so using this transformer on values sourced from an RDF model is not recommended — "index 0" +against such a sequence would be arbitrary, and could change between runs. + +If `emptyStringToEmptyResult` is enabled, a result that is an empty string is treated the same as no result at +all, rather than being returned as an empty string. + +## Example usage + +Given the input sequence `a, b, c`: + +* Index `-1` returns `c` — the last value. +* Index `-3` — equal to the negative length of the sequence — wraps around and returns `a`, the first value. + +Given the single-element input sequence `a`: + +* Index `-2` is past the start of the sequence, so by default the result is empty. +* Index `-2` with `failIfNotFound` enabled throws an error instead of returning an empty result. + +## Examples + +**Notation:** List of values are represented via square brackets. Example: `[first, second]` represents a list of two values "first" and "second". + +--- +**A negative index counts from the end; -1 returns the last value:** + +* Parameters + * index: `-1` + +* Input values: + 1. `[a, b, c]` + +* Returns: `[c]` + + +--- +**An index equal to the negative sequence length wraps to the first value:** + +* Parameters + * index: `-3` + +* Input values: + 1. `[a, b, c]` + +* Returns: `[a]` + + +--- +**A negative index past the start of the sequence returns an empty result by default:** + +* Parameters + * index: `-2` + +* Input values: + 1. `[a]` + +* Returns: `[]` + + +--- +**A negative index past the start of the sequence throws if 'failIfNotFound' is enabled:** + +* Parameters + * index: `-2` + * failIfNotFound: `true` + +* Input values: + 1. `[a]` + +* Returns: `[]` +* **Throws error:** `IndexOutOfBoundsException` - If emptyStringToEmptyResult is true then instead of a result with an empty String, an empty result is returned. @@ -24,31 +106,31 @@ Returns the value found at the specified index. Fails or returns an empty result ### Index -No description +The index of the value: 0-based if non-negative; if negative, counts from the end, wrapping at -length. -- ID: `index` -- Datatype: `int` -- Default Value: `None` +* ID: `index` +* Datatype: `int` +* Default Value: `None` ### Fail if not found -No description +If enabled, the transformation fails when no value exists at the given index; otherwise it is dropped. -- ID: `failIfNotFound` -- Datatype: `boolean` -- Default Value: `false` +* ID: `failIfNotFound` +* Datatype: `boolean` +* Default Value: `false` ### Empty string to empty result -No description +If enabled, an empty-string result is treated as no result, instead of being returned as an empty string. -- ID: `emptyStringToEmptyResult` -- Datatype: `boolean` -- Default Value: `false` +* ID: `emptyStringToEmptyResult` +* Datatype: `boolean` +* Default Value: `false` ## Advanced Parameter diff --git a/docs/build/reference/transformer/Template/TemplateTransformer.md b/docs/build/reference/transformer/Template/TemplateTransformer.md index 763677de4..b8c7bafb3 100644 --- a/docs/build/reference/transformer/Template/TemplateTransformer.md +++ b/docs/build/reference/transformer/Template/TemplateTransformer.md @@ -1,6 +1,6 @@ --- title: "Evaluate template" -description: "Evaluates a template. Input values can be addressed using the variables 'input1', 'input2', etc. Global variables are available in the 'global' scope, e.g., 'global.myVar'." +description: "Evaluates a template. Input values can be addressed using the variables 'input1', 'input2', etc. Global variables are available in the 'global' scope, e.g., 'global.myVar', except for sensitive ones." icon: octicons/cross-reference-24 tags: - TransformOperator @@ -12,7 +12,7 @@ tags: -Evaluates a template. Input values can be addressed using the variables 'input1', 'input2', etc. Global variables are available in the 'global' scope, e.g., 'global.myVar'. +Evaluates a template. Input values can be addressed using the variables 'input1', 'input2', etc. Global variables are available in the 'global' scope, e.g., 'global.myVar', except for sensitive ones. ## Examples diff --git a/docs/build/reference/transformer/Uncategorized/cmem-plugin-jq-transform.md b/docs/build/reference/transformer/Uncategorized/cmem-plugin-jq-transform.md index 0de234ce3..b3fdc7dad 100644 --- a/docs/build/reference/transformer/Uncategorized/cmem-plugin-jq-transform.md +++ b/docs/build/reference/transformer/Uncategorized/cmem-plugin-jq-transform.md @@ -18,6 +18,27 @@ tags: e.g. with cmemc. +This operator applies a jq filter to each value of a single input path and +emits the filter's result as the value on the operator's output path. + +Exactly one input is required - the operator fails if it is connected to no +input path or to more than one. Each value of that path is passed to the +filter independently, so a multi-valued path produces one filter run, and one +output value, per input value. + +A filter run can emit zero, one or several JSON values. If it emits exactly +one and **Output list with one item as string** is enabled (the default), +that single value is converted with Python's `str()`. For a plain JSON +string this yields the text unchanged, but for any other JSON type - numbers, +booleans, `null`, objects, arrays - it yields Python's own literal syntax +instead of JSON, for example `true` becomes `True` and `null` becomes `None`. +The same applies, regardless of the option, whenever a run emits zero or +several values: the operator then outputs the Python `str()` of the whole +result list. Filters that are expected to always return a single JSON string +are unaffected; filters that can return other types or a variable number of +results will need their output parsed back from this Python-literal form +downstream. + > [jq](https://jqlang.org/) is like sed for JSON data - you can use it to > slice and filter and map and transform structured data with the same ease that sed, awk, > grep and friends let you play with text. @@ -92,7 +113,7 @@ The jq program to apply to the input JSON string. ### Output list with one item as string - +If enabled, a filter result of exactly one value is written as that value's Python string representation instead of as a one-element list. If disabled, or if the filter returns zero or several values, the whole result list is written the same way. - ID: `single_item_as_string` - Datatype: `boolean` diff --git a/docs/build/reference/transformer/Uncategorized/cmem_plugin_currencies-transform.md b/docs/build/reference/transformer/Uncategorized/cmem_plugin_currencies-transform.md index 2bedffc5c..145db9d8a 100644 --- a/docs/build/reference/transformer/Uncategorized/cmem_plugin_currencies-transform.md +++ b/docs/build/reference/transformer/Uncategorized/cmem_plugin_currencies-transform.md @@ -73,7 +73,7 @@ Set date (e.g.YYYY-MM-DD) to convert currencies based on historic rates. - ID: `date` - Datatype: `string` -- Default Value: `2026-09-02` +- Default Value: `2026-11-11` diff --git a/docs/build/reference/transformer/Validation/validateRegex.md b/docs/build/reference/transformer/Validation/validateRegex.md index 7369e2a6b..0e6459aa6 100644 --- a/docs/build/reference/transformer/Validation/validateRegex.md +++ b/docs/build/reference/transformer/Validation/validateRegex.md @@ -140,3 +140,4 @@ regular expression * [ifMatchesRegex](../Conditional/ifMatchesRegex.md) — A regular expression match plays different roles here. The Validate regex plugin checks each value against the pattern and passes it through only when it fully matches. The If matches regex plugin uses the match to choose which provided branch value is returned. * [regexSelect](../Selection/regexSelect.md) — Regex selection turns one checked value and a list of patterns into a result sequence aligned with that list, placing the provided output value wherever a pattern matches. Validate regex keeps the original value and treats the pattern as a full-value check. * [regexExtract](../Extract/regexExtract.md) — Regex extract turns the match into output by returning the matched substring or the first capturing group. Validate regex leaves the value unchanged and only lets it through when the full value matches the pattern. +* [filterByRegex](../Filter/filterByRegex.md) — A non-matching value is excluded by Filter by regex; every other value continues. One mismatch fails the whole check under Validate regex. diff --git a/docs/build/reference/transformer/Value/inputHash.md b/docs/build/reference/transformer/Value/inputHash.md index 983a867cc..ccdc6ce9c 100644 --- a/docs/build/reference/transformer/Value/inputHash.md +++ b/docs/build/reference/transformer/Value/inputHash.md @@ -1,6 +1,6 @@ --- title: "Combined input hash" -description: "Calculates a single hash value covering all input values combined, across all input ports. Values are fed into the hash function in port order without any separator between them." +description: "Calculates a single hash value covering all input values combined, across all input ports. Values are fed into the hash function in port order, with an optional separator (the glue parameter) inserted between them." icon: octicons/cross-reference-24 tags: - TransformOperator @@ -12,35 +12,43 @@ tags: -The **Combined input hash** operator produces exactly one hash value covering all input values combined, across all connected input ports. However many values arrive and however many ports are connected, the output is always a single string. +The **Combined input hash** operator produces exactly one hash value covering all input values combined, across all connected input ports, with an optional separator inserted between them. ## How combining works -All values from all input ports are fed sequentially into a single hash function — port 1 first, then port 2, and so on. Within each port, values are processed in the order they arrive. No separator is inserted between values or between ports. The hash covers the concatenated byte content of all values in that traversal order. +All values from all input ports are fed sequentially into a single hash function, in port order; within each port, values are processed in the order they arrive. An optional separator, set with the glue parameter, can be inserted between every adjacent pair of values in that order — see the Glue parameter section below. The hash covers the concatenated byte content of all values in that order, together with any inserted separator. -This means the result depends on both the content and the order of values. The same set of values in a different order produces a different hash. Connecting one port with values `["apple", "banana"]` produces the same hash as connecting two ports with `["apple"]` and `["banana"]` respectively, because the bytes are fed in the same sequence either way. +Swapping two values, or changing a single character within one value, produces a different hash. Connecting one port with values `["apple", "banana"]` produces the same hash as connecting two ports with `["apple"]` and `["banana"]` respectively, because the bytes are fed in the same sequence either way. ## Output -The output is a single lowercase hexadecimal string. The length depends on the algorithm: 64 characters for SHA-256, 32 for MD5, 40 for SHA-1, 96 for SHA-384, 128 for SHA-512. If the input is empty, the output is the hash of an empty message. +The output is a single lowercase hexadecimal string whose length is fixed by the chosen algorithm: 64 characters for SHA-256, 32 for MD5, 40 for SHA-1, 96 for SHA-384, 128 for SHA-512. Connecting no values at all still produces a hash — the hash of the empty message, not an error. -Values are encoded as UTF-8 before hashing. +Values are encoded as UTF-8 before hashing, so the same text produces the same bytes regardless of platform or locale. ## Algorithm parameter -The algorithm parameter selects the hash function. The default is SHA-256. The following algorithms from the [SPARQL 1.1 specification](https://www.w3.org/TR/sparql11-query/#func-hash) are supported: +The algorithm parameter selects the hash function. The following algorithms from the [SPARQL 1.1 specification](https://www.w3.org/TR/sparql11-query/#func-hash) are supported: | SPARQL name | Java name | Notes | |-------------|-----------|-------| -| MD5 | MD5 | Weak — vulnerable to collision attacks. Avoid for security-sensitive use. | -| SHA1 | SHA-1 | Weak — deprecated for most security purposes. | -| SHA256 | SHA-256 | Recommended default. | -| SHA384 | SHA-384 | Stronger than SHA-256. | -| SHA512 | SHA-512 | Strongest in the SPARQL set. | +| MD5 | MD5 | Has known collision vulnerabilities. | +| SHA1 | SHA-1 | Deprecated for most security purposes. | +| SHA256 | SHA-256 | Default. | +| SHA384 | SHA-384 | | +| SHA512 | SHA-512 | | Additional algorithms available on the JVM (such as SHA-512/256 and SHA-3 variants) are also accepted. The full list is JVM-dependent and visible in the algorithm parameter dropdown. -Note that the Java names use hyphens (SHA-256, SHA-1) where SPARQL uses none (SHA256, SHA1). Both forms are accepted by this operator. +The Java names use hyphens (SHA-256, SHA-1) where SPARQL uses none (SHA256, SHA1); both forms are accepted by this operator. + +## Glue parameter + +The glue parameter sets a separator string that is inserted between every adjacent pair of values in the traversal order described above, defaulting to an empty string. Its text can contain the escaped sequences `\n`, `\t`, and `\\`, which are converted to a newline, a tab, and a backslash respectively before hashing. + +Connecting one port with values `["apple", "banana"]` and a glue of `-` produces the same hash as connecting two ports with `["apple"]` and `["banana"]` respectively, using the same glue, because the glue is inserted at the same point in the byte sequence either way. + +An input port without a value is skipped, and the glue is only inserted between the values that exist. To insert the glue for such a port, the port can be connected through a Default Value operator with an empty value. ## Examples @@ -65,7 +73,7 @@ Note that the Java names use hyphens (SHA-256, SHA-1) where SPARQL uses none (SH --- -**Reversing the value order produces a different hash, confirming order-sensitivity:** +**The order in which values are combined affects the resulting hash:** * Input values: 1. `[banana, apple]` @@ -74,7 +82,7 @@ Note that the Java names use hyphens (SHA-256, SHA-1) where SPARQL uses none (SH --- -**Values from multiple ports are combined in port order, producing the same hash as the equivalent single-port sequence:** +**Values from multiple ports are combined in port order: port 1's values first, then port 2's:** * Input values: 1. `[apple]` @@ -83,6 +91,262 @@ Note that the Java names use hyphens (SHA-256, SHA-1) where SPARQL uses none (SH * Returns: `[5b692305517af54eb5ae12b9ff89eaf89e31f6a6ee208365886a18b81a2fc2f8]` +--- +**The glue parameter inserts a separator between values:** + +* Parameters + * glue: `-` + +* Input values: + 1. `[apple, banana]` + +* Returns: `[d6300f978b8699862450cbabc35349239c09e95c356723c32fc1407ec8104c78]` + + +--- +**Values from multiple ports are combined using the glue separator in port order: port 1's values first, then port 2's:** + +* Parameters + * glue: `-` + +* Input values: + 1. `[apple]` + 2. `[banana]` + +* Returns: `[d6300f978b8699862450cbabc35349239c09e95c356723c32fc1407ec8104c78]` + + +--- +**The glue parameter accepts escaped characters, here a newline:** + +* Parameters + * glue: `\n` + +* Input values: + 1. `[apple, banana]` + +* Returns: `[5c573825e0a168e9e382f9b7e78e6603271634dceac792b9c92fa77a27d404f8]` + + +--- +**Values containing non-ASCII characters are encoded as UTF-8 before being hashed:** + +* Parameters + * glue: `-` + +* Input values: + 1. `[café, banana]` + +* Returns: `[c17ffffc64b5ffd83dbc2a975134c65a7f7f457a898b6e1fc6d2db1217fa92ff]` + + +--- +**The glue parameter accepts a separator longer than one character:** + +* Parameters + * glue: `::` + +* Input values: + 1. `[apple, banana]` + +* Returns: `[7f721cdbf970bad79a82e7d827e0e5eef3d8263610aa6f062427060f5d2379db]` + + +--- +**The order of values combined with a glue separator affects the resulting hash:** + +* Parameters + * glue: `-` + +* Input values: + 1. `[banana, apple]` + +* Returns: `[4472ed905ebe09b0e7a37ffa5adc6b0b2f052a6d0045fc037e2e56cdb45fba0a]` + + +--- +**The glue separator is never applied to a single value, since there is no adjacent value to separate it from:** + +* Parameters + * glue: `-` + +* Input values: + 1. `[solo]` + +* Returns: `[5364f2f2fc4f54e9d47ad29cfb08ef430c8153394bf2a0dff5cbe77a0ffef861]` + + +--- +**An input port without a value is skipped, and no glue is inserted for it:** + +* Parameters + * glue: `-` + +* Input values: + 1. `[]` + 2. `[solo]` + +* Returns: `[5364f2f2fc4f54e9d47ad29cfb08ef430c8153394bf2a0dff5cbe77a0ffef861]` + + +--- +**An empty port 2 is skipped in the same way, producing the same hash as an empty port 1:** + +* Parameters + * glue: `-` + +* Input values: + 1. `[solo]` + 2. `[]` + +* Returns: `[5364f2f2fc4f54e9d47ad29cfb08ef430c8153394bf2a0dff5cbe77a0ffef861]` + + +--- +**Without any input values, the glue is not hashed on its own, producing the hash of an empty message:** + +* Parameters + * glue: `-` + +* Input values: + 1. `[]` + 2. `[]` + +* Returns: `[e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855]` + + +--- +**With more than two values, the glue separator is inserted between every adjacent pair:** + +* Parameters + * glue: `-` + +* Input values: + 1. `[a, b]` + 2. `[c]` + +* Returns: `[cbd2be7b96f770a0326948ebd158cf539fab0627e8adbddc97f7a65c6a8ae59a]` + + +--- +**Three values in one port produce a hash with the glue separator inserted between every pair:** + +* Parameters + * glue: `-` + +* Input values: + 1. `[a, b, c]` + +* Returns: `[cbd2be7b96f770a0326948ebd158cf539fab0627e8adbddc97f7a65c6a8ae59a]` + + +--- +**A value that is identical to the glue string is hashed as ordinary byte content, with no special escaping:** + +* Parameters + * glue: `-` + +* Input values: + 1. `[a, -, b]` + +* Returns: `[1acd27f89b4f44ddaf3b81fadb5491ea61f9ae3981545cf62d3502a387bcf350]` + + +--- +**Values from three ports are combined using the glue separator in port order: port 1's value, then port 2's, then port 3's:** + +* Parameters + * glue: `-` + +* Input values: + 1. `[a]` + 2. `[b]` + 3. `[c]` + +* Returns: `[cbd2be7b96f770a0326948ebd158cf539fab0627e8adbddc97f7a65c6a8ae59a]` + + +--- +**The order in which values are combined affects the resulting hash, even with more than two values and a glue separator:** + +* Parameters + * glue: `-` + +* Input values: + 1. `[c, b]` + 2. `[a]` + +* Returns: `[8e599734c8f75bd7866757e5f9cd4ab3cc173d753fb858a85ae6efb817e3e200]` + + +--- +**With more than two values, the SHA-1 algorithm still inserts the glue separator between every adjacent pair:** + +* Parameters + * algorithm: `SHA-1` + * glue: `-` + +* Input values: + 1. `[a, b]` + 2. `[c]` + +* Returns: `[e088d9e3f737c091378fe8494936b16d51eb42ee]` + + +--- +**With the SHA-512 algorithm selected, the glue separator is inserted between every adjacent pair of three values:** + +* Parameters + * algorithm: `SHA-512` + * glue: `-` + +* Input values: + 1. `[a, b]` + 2. `[c]` + +* Returns: `[a271a54e93179f7dccb132a17c914e66e76a91ae3b7cb8b3abffa21a8ffb66b65a2883769bc4c6f95ab39fbd039cfe21046b485cef1588269d2605c7253f2b98]` + + +--- +**With the MD5 algorithm selected, the glue parameter still inserts a separator between values:** + +* Parameters + * algorithm: `MD5` + * glue: `-` + +* Input values: + 1. `[apple, banana]` + +* Returns: `[9b22c402a5166c71ebdf2d1b99f37727]` + + +--- +**With the MD5 algorithm selected, escaped glue characters are still converted before being used as the separator:** + +* Parameters + * algorithm: `MD5` + * glue: `\n` + +* Input values: + 1. `[apple, banana]` + +* Returns: `[cf0bf238cadd6688db6f8d2511a52ebf]` + + +--- +**With the MD5 algorithm and a glue of '-', the hash equals the SPARQL expression MD5(CONCAT(?a, "-", ?b)):** + +* Parameters + * algorithm: `MD5` + * glue: `-` + +* Input values: + 1. `[a]` + 2. `[b]` + +* Returns: `[8ca2ed590cf2ea2404f2e67641bcdf50]` + + --- **The algorithm parameter selects the hash function (MD5):** @@ -165,6 +429,16 @@ The hash algorithm to be used. * Datatype: `string` * Default Value: `SHA256` + + +### Glue + +Separator to be inserted between two concatenated strings. The text can contain escaped characters \n, \t and \\ that are replaced by a newline, tab or backslash respectively. + +* ID: `glue` +* Datatype: `string` +* Default Value: `None` + ## Advanced Parameter `None` @@ -173,3 +447,6 @@ The hash algorithm to be used. * [perValueHash](perValueHash.md) — The Per-value hash plugin hashes each input value independently and returns one hash per value, preserving cardinality. The Combined input hash plugin instead feeds all values into a single hash function, producing one combined hash regardless of input size. * [mapWithDefaultInput](../Replace/mapWithDefaultInput.md) — One hash value is produced for the entire set of inputs by the Combined input hash plugin. The Map with default plugin instead keeps a value sequence and rewrites it position by position through the mapping, falling back to the second input where no mapping entry is found. +* [concat](../Combine/concat.md) — Combined input hash always returns exactly one value, however many values or ports are connected. Concatenate instead produces every combination of connected values as a separate output, so more than one value on an input produces multiple outputs instead of one. +* [zip](../Combine/zip.md) — Combined input hash accepts any number of ports and always returns exactly one value. Zip instead requires exactly two inputs, pairs their values by position, and returns one output per position rather than one combined result. +* [concatMultiValues](../Combine/concatMultiValues.md) — Concatenate multiple values combines everything received on one port into a single string, but keeps each port's result separate; more ports still return separate results, one per port. Combined input hash instead crosses every port boundary, always returning one hash value overall. diff --git a/docs/build/reference/transformer/index.md b/docs/build/reference/transformer/index.md index a9e31201b..1e0d6c72f 100644 --- a/docs/build/reference/transformer/index.md +++ b/docs/build/reference/transformer/index.md @@ -22,7 +22,7 @@ Transform operators transform a one or more sequences of string values to a sequ | [Capitalize](Normalize/capitalize.md) | Normalize | Capitalizes the string i.e. converts the first character to upper case. If 'allWords' is set to true, all words are capitalized and not only the first character. | | [Clean HTML](Normalize/htmlCleaner.md) | Normalize | Cleans HTML markup using a tag whitelist and allows selection of HTML sections with XPath or CSS selector expressions. | | [Coalesce (first non-empty input)](Selection/coalesce.md) | Selection | Forwards the first non-empty input, i.e. for which any value(s) exist. A single empty string is considered a value. | - | [Combined input hash](Value/inputHash.md) | Value | Calculates a single hash value covering all input values combined, across all input ports. Values are fed into the hash function in port order without any separator between them. | + | [Combined input hash](Value/inputHash.md) | Value | Calculates a single hash value covering all input values combined, across all input ports. Values are fed into the hash function in port order, with an optional separator (the glue parameter) inserted between them. | | [Compare dates](Date/compareDates.md) | Date | Compares two dates. | | [Compare numbers](Numeric/compareNumbers.md) | Numeric | Compares the numbers of two sets. Returns 1 if the comparison yields true and 0 otherwise. If there are multiple numbers in both sets, the comparator must be true for all numbers. For instance, {1,2} < {2,3} yields 0 as not all numbers in the first set are smaller than in the second. | | [Concatenate](Combine/concat.md) | Combine | Concatenates strings from multiple inputs. | @@ -48,7 +48,7 @@ Transform operators transform a one or more sequences of string values to a sequ | [Encode URL](Normalize/urlEncode.md) | Normalize | URL encodes the string. | | [Escape SPARQL multiline literal](SPARQL/escape_multiline_literal.md) | SPARQL | Escapes a value so it can be safely used inside a SPARQL triple-quoted string literal (`"""..."""` or `'''...'''`). Escapes backslashes and breaks any run of three or more consecutive single or double quotes. Individual quotes and newlines are preserved. The returned value does not include enclosing quotation marks. | | [Escape SPARQL plain literal](SPARQL/escape_literal.md) | SPARQL | Escapes a value so it can be safely used inside a SPARQL short-form string literal. Escapes backslashes, quotes, newlines, carriage returns and tabs. The returned value does not include enclosing quotation marks. | - | [Evaluate template](Template/TemplateTransformer.md) | Template | Evaluates a template. Input values can be addressed using the variables 'input1', 'input2', etc. Global variables are available in the 'global' scope, e.g., 'global.myVar'. | + | [Evaluate template](Template/TemplateTransformer.md) | Template | Evaluates a template. Input values can be addressed using the variables 'input1', 'input2', etc. Global variables are available in the 'global' scope, e.g., 'global.myVar', except for sensitive ones. | | [Excel Abs](Excel/Excel_ABS.md) | Excel | Excel ABS(number): Returns the absolute value of the given number. | | [Excel Acos](Excel/Excel_ACOS.md) | Excel | Excel ACOS(number): Returns the inverse cosine of the given number in radians. | | [Excel Acosh](Excel/Excel_ACOSH.md) | Excel | Excel ACOSH(number): Returns the inverse hyperbolic cosine of the given number in radians. | @@ -166,10 +166,10 @@ Transform operators transform a one or more sequences of string values to a sequ | [Extract physical quantity](Numeric/extractPhysicalQuantity.md) | Numeric | Extracts physical quantities, such as length or weight values. Values are expected to be formatted as `{Number}{UnitPrefix}{Symbol}` and are converted to the base unit. | | [File hash](Metadata/fileHash.md) | Metadata | Calculates the hash sum of a file. The hash sum is cached so that subsequent calls to this operator are fast. Note that initially and every time the specified resource has been updated, this operator might take a long time (depending on the file size). This operator supports using different hash algorithms from the [Secure Hash Algorithms family](https://en.wikipedia.org/wiki/Secure_Hash_Algorithms) (SHA, e.g. SHA256) and two algorithms from the [Message-Digest Algorithm family](https://en.wikipedia.org/wiki/MD5) (MD2 / MD5). Please be aware that some of these algorithms are not secure regarding collision- and other attacks. Note: This transform operator ignores any inputs. | | [Filter by length](Filter/filterByLength.md) | Filter | Removes all strings that are shorter than 'min' characters and longer than 'max' characters. | - | [Filter by regex](Filter/filterByRegex.md) | Filter | Removes all strings that do NOT match a regex. If 'negate' is true, only strings will be removed that match the regex. | + | [Filter by regex](Filter/filterByRegex.md) | Filter | This transformer filters values by a regular expression: values matching the regex are kept, unless 'negate' is set, which keeps only the values that don't. By default the regex must match a value in full; setting 'contains' relaxes that so any value containing a match is kept instead. | | [Fix URI](Normalize/uriFix.md) | Normalize | Generates valid absolute URIs from the given values. Already valid absolute URIs are left untouched. | | [Format number](Numeric/formatNumber.md) | Numeric | Formats a number according to a user-defined pattern. The pattern syntax is documented at: | - | [Get value by index](Sequence/getValueByIndex.md) | Sequence | Returns the value found at the specified index. Fails or returns an empty result depending on failIfNoFound is set or not. Please be aware that this will work only if the data source supports some kind of ordering like XML or JSON. This is probably not a good idea to do with RDF models. If emptyStringToEmptyResult is true then instead of a result with an empty String, an empty result is returned. | + | [Get value by index](Sequence/getValueByIndex.md) | Sequence | Returns the value found at the specified index. A negative index counts from the end of the sequence. | | [If contains](Conditional/ifContains.md) | Conditional | Accepts two or three inputs. If the first input contains the given value, the second input is forwarded. Otherwise, the third input is forwarded (if present). | | [If exists](Conditional/ifExists.md) | Conditional | Accepts two or three inputs. If the first input provides a value, the second input is forwarded. Otherwise, the third input is forwarded (if present). | | [If matches regex](Conditional/ifMatchesRegex.md) | Conditional | This transformer uses a regular expression as a matching condition, in order to distinguish which input to take. | diff --git a/docs/build/spark/index.md b/docs/build/spark/index.md index 1fad411cf..b89780f0a 100644 --- a/docs/build/spark/index.md +++ b/docs/build/spark/index.md @@ -4,11 +4,11 @@ tags: - Introduction - Explainer --- -# Apache Spark within Corporate Memory Build +# Apache Spark within eccenca Corporate Memory Build ## Introduction -This documentation provides an overview of Apache Spark and its integration within Corporate Memory’s Build component. +This documentation provides an overview of Apache Spark and its integration within eccenca Corporate Memory’s Build component. The goal is to provide a conceptual understanding of Spark, its purpose in Build, and how workflows leverage Spark-aware datasets for efficient, distributed data processing. The documentation is structured in two parts: diff --git a/docs/build/tutorial-how-to-link-ids-to-osint/define-the-interfaces/index.md b/docs/build/tutorial-how-to-link-ids-to-osint/define-the-interfaces/index.md index c936fd293..acfd1ebd2 100644 --- a/docs/build/tutorial-how-to-link-ids-to-osint/define-the-interfaces/index.md +++ b/docs/build/tutorial-how-to-link-ids-to-osint/define-the-interfaces/index.md @@ -38,7 +38,7 @@ Here, the figure 3 is nice but before this first schema during the project, ther ## Technical feasibility with the available information It's not really technical to check if the data is available or not, but without technical knowledge, it's hard to evaluate the difficulty to link each id to their instance in your RDF knowledge graph. -In this tutorial, we learn to use Corporate Memory of Eccenca to transform these IDs to IRI to import properly these ID with the other data necessary to build these interface. +In this tutorial, we learn to use eccenca Corporate Memory to transform these IDs to IRI to import properly these ID with the other data necessary to build these interface. After research and one meeting with analysts, we have chosen the datasets of Mitre Attack, the datasets of IoC rules (Sigma and Hayabusa) in Github and of course, the IoCs data already in the Splunk indexes. @@ -47,7 +47,7 @@ After research and one meeting with analysts, we have chosen the datasets of Mit
Figure 5. Define the information available in alerts of IDS and in OSINT to link these information.
-The Splunk indexes of IoCs are selected by the analysts in the dashboard via the component [multiselect input](https://docs.splunk.com/Documentation/Splunk/9.0.5/Viz/FormEditor#Multiselect) in the form part of dashboard (the form part inits other Splunk tokens). We have choosen the IDs to link these data and the figure 5 resumes how we are going to link these data via Corporate Memory of Eccenca. +The Splunk indexes of IoCs are selected by the analysts in the dashboard via the component [multiselect input](https://docs.splunk.com/Documentation/Splunk/9.0.5/Viz/FormEditor#Multiselect) in the form part of dashboard (the form part inits other Splunk tokens). We have choosen the IDs to link these data and the figure 5 resumes how we are going to link these data via eccenca Corporate Memory. ## Understand the base of Splunk dashboards diff --git a/docs/build/tutorial-how-to-link-ids-to-osint/define-the-need/index.md b/docs/build/tutorial-how-to-link-ids-to-osint/define-the-need/index.md index 903e7e13e..d6db11373 100644 --- a/docs/build/tutorial-how-to-link-ids-to-osint/define-the-need/index.md +++ b/docs/build/tutorial-how-to-link-ids-to-osint/define-the-need/index.md @@ -16,7 +16,7 @@ There are three things to know to describe a need: 2. What 3. The context -With our customers, we claim a knowledge graph can reduce the work of analysts in the Cybersecurity. So, one of our customer wanted to help their IN Analysts with the software Corporate Memory of eccenca. +With our customers, we claim a knowledge graph can reduce the work of analysts in the Cybersecurity. So, one of our customer wanted to help their IN Analysts with the software eccenca Corporate Memory. After several meeting, we defined the need. ### For who @@ -64,7 +64,7 @@ Here, the context shows this project is an exploration with like deliverable pro During this step of R&D, we are developing the tools to request a SPARQL service via an authentification OAuth directly by Splunk dashboard. You will install these Splunk apps step-by-step during this tutorial or you can download their "tar.gz" in SPlunk now: - "[Linked Data App](../link-IDS-event-to-KG/eccenca_commands.tar.gz)" to install the SPARQL command. You need to configure it with the details in his README file or in [its tutorial page](../link-IDS-event-to-KG/index.md). -- "[Investigate lateral movements with a knowledge graph](../link-IDS-event-to-KG-via-cmem/eccenca_poc_investigate.tar.gz)" to install our demonstrator to connect SPLUNK to CMEM. You need to read his README or [its tutorial page](../link-IDS-event-to-KG/index.md) to understand this example. +- "[Investigate lateral movements with a knowledge graph](../link-IDS-event-to-KG-via-cmem/eccenca_poc_investigate.tar.gz)" to install our demonstrator to connect SPLUNK to eccenca Corporate Memory. You need to read his README or [its tutorial page](../link-IDS-event-to-KG/index.md) to understand this example. After several propositions, analysts oriented the implementation of our first dashboards and finally, we showed clearly a benefice to use a knowledge graph via these results. diff --git a/docs/build/tutorial-how-to-link-ids-to-osint/index.md b/docs/build/tutorial-how-to-link-ids-to-osint/index.md index 94bd82722..00d3c5910 100644 --- a/docs/build/tutorial-how-to-link-ids-to-osint/index.md +++ b/docs/build/tutorial-how-to-link-ids-to-osint/index.md @@ -17,7 +17,7 @@ tags: 2. [Indicators of compromise rules, like Hayabusa and Sigma rules](lift-data-from-YAML-data-of-hayabusa-sigma/index.md) 5. Link IDS event to a knowledge graph in dashboards via: 1. [Queries](link-IDS-event-to-KG/index.md) - 2. [Inferences](link-IDS-event-to-KG-via-cmem/index.md) (for the advanced users of Corporate Memory) + 2. [Inferences](link-IDS-event-to-KG-via-cmem/index.md) (for the advanced users of eccenca Corporate Memory) ## Introduction diff --git a/docs/build/tutorial-how-to-link-ids-to-osint/lift-data-from-STIX-2.1-data-of-mitre-attack/index.md b/docs/build/tutorial-how-to-link-ids-to-osint/lift-data-from-STIX-2.1-data-of-mitre-attack/index.md index 22e50d0df..ed909111f 100644 --- a/docs/build/tutorial-how-to-link-ids-to-osint/lift-data-from-STIX-2.1-data-of-mitre-attack/index.md +++ b/docs/build/tutorial-how-to-link-ids-to-osint/lift-data-from-STIX-2.1-data-of-mitre-attack/index.md @@ -14,7 +14,7 @@ The MITRE ATT&CK datasets in STIX 2.1 JSON collections are here: The "ontology" of MITRE ATT&CK with STIX is here: [https://github.com/mitre/cti/blob/master/USAGE.md](https://github.com/mitre/cti/blob/master/USAGE.md) -The objective of this tutorial is not focus on the ontologies. In our use case, we just need to extract several metadata. If the community of STIX wants to resolve their problems, it will be the moment to define a solid ontology. If you are a newbie with the Linked Data technologies, you have to learn to generate in first a functional knowledge graph for your needs before building a perfect ontology for everybody. When you masterize the ontologies, we will modify this first ontology and you could refresh your knowledge graph when you want with Corporate Memory. +The objective of this tutorial is not focus on the ontologies. In our use case, we just need to extract several metadata. If the community of STIX wants to resolve their problems, it will be the moment to define a solid ontology. If you are a newbie with the Linked Data technologies, you have to learn to generate in first a functional knowledge graph for your needs before building a perfect ontology for everybody. When you masterize the ontologies, we will modify this first ontology and you could refresh your knowledge graph when you want with eccenca Corporate Memory. This tutorial is written in order to gradually acquire all the skills necessary to build from scratch a knowledge graph with Corporate Memory and update it automatically via Corporate Memory Console. This tutorial must be completed in order. @@ -626,9 +626,9 @@ Here, we are creating a new SPARQL Update task to calculate and insert automatic ### Refresh all automatically -The datasets of Mitre are updated regularly. You may want to update them automatically via a command line in a bash file. In this script, we use CMEM. +The datasets of Mitre are updated regularly. You may want to update them automatically via a command line in a bash file. In this script, we use eccenca Corporate Memory. -1. [Install CMEMC - a Command Line Interface of CMEM](../../../automate/cmemc-command-line-interface/installation/index.md) +1. [Install cmemc - a Command Line Interface of Corporate Memory](../../../automate/cmemc-command-line-interface/installation/index.md) 2. Open your config file: @@ -636,7 +636,7 @@ The datasets of Mitre are updated regularly. You may want to update them automat cmemc config edit ``` -3. Insert your sandbox in your CMEMC config, example with a password grant type: +3. Insert your sandbox in your cmemc config, example with a password grant type: ```bash [johndo.eccenca.my] @@ -651,7 +651,7 @@ The datasets of Mitre are updated regularly. You may want to update them automat !!! Tip - Immediatly, in the file ~/.bashrc, you can specify your sandbox like your instance by default for CMEMC with this line: + Immediatly, in the file ~/.bashrc, you can specify your sandbox like your instance by default for cmemc with this line: ```bash export CMEMC_CONNECTION=johndo.eccenca.my @@ -788,7 +788,7 @@ STIX uses JSON syntax and can therefore be converted to RDF via Corporate Memory ## Ressources * [RDF schemas (Model, pattern, etc)](RDF_model_and_pattern.drawio) -* [Archive of CMEM project](MITREATTCK_tutorial.zip) +* [Archive of Corporate Memory project](MITREATTCK_tutorial.zip) --- diff --git a/docs/build/tutorial-how-to-link-ids-to-osint/lift-data-from-YAML-data-of-hayabusa-sigma/index.md b/docs/build/tutorial-how-to-link-ids-to-osint/lift-data-from-YAML-data-of-hayabusa-sigma/index.md index 08db384d9..fc6cb4e70 100644 --- a/docs/build/tutorial-how-to-link-ids-to-osint/lift-data-from-YAML-data-of-hayabusa-sigma/index.md +++ b/docs/build/tutorial-how-to-link-ids-to-osint/lift-data-from-YAML-data-of-hayabusa-sigma/index.md @@ -11,7 +11,7 @@ Here, we are working with the Hayabusa/Sigma rules available via GitHub: - [https://github.com/Yamato-Security/hayabusa-rules](https://github.com/Yamato-Security/hayabusa-rules) - [https://github.com/SigmaHQ/sigma](https://github.com/Yamato-Security/hayabusa-rules) -The problem of interoperability, here, is the YAML format of files, their random position in their folders in their Github projets. Moreover, the same rule can exist in different projects but in this tutorial, we will not fix this problem and we consider the IRI rule is their Web address. In Corporate Memory, we would fix that with the Linked Tool, we will study this tool in a next part of this tutorial. +The problem of interoperability, here, is the YAML format of files, their random position in their folders in their Github projets. Moreover, the same rule can exist in different projects but in this tutorial, we will not fix this problem and we consider the IRI rule is their Web address. In eccenca Corporate Memory, we would fix that with the Linked Tool, we will study this tool in a next part of this tutorial. To build this knowledge graph of rules, we need to: diff --git a/docs/build/tutorial-how-to-link-ids-to-osint/link-IDS-event-to-KG-via-cmem/index.md b/docs/build/tutorial-how-to-link-ids-to-osint/link-IDS-event-to-KG-via-cmem/index.md index 2e49803f6..7d52e3857 100644 --- a/docs/build/tutorial-how-to-link-ids-to-osint/link-IDS-event-to-KG-via-cmem/index.md +++ b/docs/build/tutorial-how-to-link-ids-to-osint/link-IDS-event-to-KG-via-cmem/index.md @@ -4,7 +4,7 @@ In this tutorial, we are using the Splunk app "Investigate lateral movements with a knowledge graph", like example. This app contains several dasboards to help the analysts to navigate in the Hayabusa or Sigma alerts before searching in the Suricata and Sysmon alerts. We hope this method can "Accelerate Cyber Threat Hunting". -In the demo of this Splunk app via the video 1, the user selects the data about one investigation via Splunk and generate a bash script to export these data via the Splunk API in tempory graphs in Corporate Memory for each investigation. +In the demo of this Splunk app via the video 1, the user selects the data about one investigation via Splunk and generate a bash script to export these data via the Splunk API in tempory graphs in eccenca Corporate Memory for each investigation.
![Video 1: Splunk dashboards of the Splunk app "Investigate lateral movements with a knowledge graph"](splunk-app-demo-poc-app.gif) @@ -139,7 +139,7 @@ There are not consensus about the manner to calculate inferences on the RDF data We use two manners to calculate new inferences: -- when the inference is simple to calculate with SPARQL, we use a SPARQL update query with parameters in Corporate Memory (like "inferenceID" to build the name of destination tempory graph). "queryIRI" is the IRI of the query in the catalog of Corporate Memory. +- when the inference is simple to calculate with SPARQL, we use a SPARQL update query with parameters in eccenca Corporate Memory (like "inferenceID" to build the name of destination tempory graph). "queryIRI" is the IRI of the query in the catalog of Corporate Memory. - when the inference is complex to calculate, we use a workflow of Corporate Memory. With these tokens "source_1", "source_2", etc and "inferences" in the dashboard, the app can generate a bash script for CMEMC. @@ -180,7 +180,7 @@ In the previous chapter, you explain the role of token "source_x" to create a ne Here the example is simple with one external token, but when you have several external tokens in the token "source_x", there are often problems to generate a complet SPL query (to check the SPL query, you need to use the Javascript console via the Javascript "investigation.js" to read the final SPL queries in the tokens "source_x"). -You can add in your dashboard several tokens "source_1", "source_2", "source_3", etc. Each source generates a part of final bash script where a curl command requests the Splunk API and a CMEMC command executes the specified worflow via its ID in the token. With this method, the analyst is free to import any Splunk data and choose any CMEM workflow according to these data. The SPL queries are executed by the Splunk API (via curl), so the analyst is free to use all commands supported in Splunk to select the data. For example, with low-level investigation, these SPL queries contains also the selected period by the analyst to limit the quantity of data to import. +You can add in your dashboard several tokens "source_1", "source_2", "source_3", etc. Each source generates a part of final bash script where a curl command requests the Splunk API and a CMEMC command executes the specified worflow via its ID in the token. With this method, the analyst is free to import any Splunk data and choose any Corporate Memory workflow according to these data. The SPL queries are executed by the Splunk API (via curl), so the analyst is free to use all commands supported in Splunk to select the data. For example, with low-level investigation, these SPL queries contains also the selected period by the analyst to limit the quantity of data to import. ## Reasoning with data in Corporate Memory via Splunk diff --git a/docs/build/tutorial-how-to-link-ids-to-osint/link-IDS-event-to-KG/index.md b/docs/build/tutorial-how-to-link-ids-to-osint/link-IDS-event-to-KG/index.md index 3e4fa775d..1bf9c7ea5 100644 --- a/docs/build/tutorial-how-to-link-ids-to-osint/link-IDS-event-to-KG/index.md +++ b/docs/build/tutorial-how-to-link-ids-to-osint/link-IDS-event-to-KG/index.md @@ -420,6 +420,6 @@ In the next page, we are using advanced tools in Coporate Memory to "Accelerate Tutorial: [how to link Intrusion Detection Systems (IDS) to Open-Source INTelligence (OSINT)](../index.md) -Next chapter: [Link IDS event to a knowledge graph in dashboards via inferences](../link-IDS-event-to-KG-via-cmem/index.md) (for the advanced users of Corporate Memory) +Next chapter: [Link IDS event to a knowledge graph in dashboards via inferences](../link-IDS-event-to-KG-via-cmem/index.md) (for the advanced users of eccenca Corporate Memory) Previous chapter: [Build a Knowledge Graph from indicators of compromise rules, like Hayabusa and Sigma rules](../lift-data-from-YAML-data-of-hayabusa-sigma/index.md) diff --git a/docs/consume/consume-graphs-in-apache-kafka/index.md b/docs/consume/consume-graphs-in-apache-kafka/index.md index a0e6d165e..fb964a4e4 100644 --- a/docs/consume/consume-graphs-in-apache-kafka/index.md +++ b/docs/consume/consume-graphs-in-apache-kafka/index.md @@ -12,7 +12,7 @@ tags: [Apache Kafka](https://kafka.apache.org/) is a distributed event store and stream-processing platform. Kafka is widely used in enterprises for data pipelines, streaming analytics, data integration and other applications. -By using the [cmem-plugin-kafka](https://pypi.org/project/cmem-plugin-kafka/) [Python Plugin](../../develop/python-plugins/index.md), you can produce and send messages to Apache Kafka from inside of our Corporate Memory Workflows. +By using the [cmem-plugin-kafka](https://pypi.org/project/cmem-plugin-kafka/) [Python Plugin](../../develop/python-plugins/index.md), you can produce and send messages to Apache Kafka from inside of our eccenca Corporate Memory Workflows. ## Installation diff --git a/docs/consume/consuming-graphs-in-power-bi/index.md b/docs/consume/consuming-graphs-in-power-bi/index.md index 25050c158..7f93a5310 100644 --- a/docs/consume/consuming-graphs-in-power-bi/index.md +++ b/docs/consume/consuming-graphs-in-power-bi/index.md @@ -8,7 +8,7 @@ tags: ## Introduction -Learn how to consume data from your Corporate Memory Knowledge Graph with our Microsoft Power-BI-Connector. +Learn how to consume data from your eccenca Corporate Memory Knowledge Graph with our Microsoft Power-BI-Connector. This manual and tutorial describes how you can consume data from your knowledge graph in Microsoft Power BI through our Corporate Memory Power-BI-Connector. diff --git a/docs/consume/index.md b/docs/consume/index.md index 44c3b5e85..d8cf95334 100644 --- a/docs/consume/index.md +++ b/docs/consume/index.md @@ -9,7 +9,7 @@ hide: # :material-star: Consume -This section outlines how to consume data from Corporate Memory Knowledge Graphs. +This section outlines how to consume data from eccenca Corporate Memory Knowledge Graphs. While there are several options to retrieve information from the Knowledge Graph, the most direct way is to issue SPARQL queries. SPARQL queries can be managed and executed in the Query Module UI. External applications may access the query catalog and execute queries through the REST API directly or more conveniently by using the [cmemc - Command Line Interface](../automate/cmemc-command-line-interface/index.md). diff --git a/docs/consume/provide-data-in-any-format-via-a-custom-api/index.md b/docs/consume/provide-data-in-any-format-via-a-custom-api/index.md index 4ac61c3d0..426d71f2b 100644 --- a/docs/consume/provide-data-in-any-format-via-a-custom-api/index.md +++ b/docs/consume/provide-data-in-any-format-via-a-custom-api/index.md @@ -9,7 +9,7 @@ tags: ## Introduction -Learn how to provide data via a customized Corporate Memory API in a text format of your choice and how to consume it in your applications. +Learn how to provide data via a customized eccenca Corporate Memory API in a text format of your choice and how to consume it in your applications. This tutorial describes how you can provide data in a text format of your choice via your own custom Corporate Memory API, and how you request those APIs. In this tutorial, we describe how you can set up an endpoint which provides [iCalendar data](https://en.wikipedia.org/wiki/ICalendar). @@ -58,7 +58,7 @@ WHERE { As a next step, we will define a template that generates iCalendar data from our previously defined SPARQL query. -Select in *Graphs* the CMEM Query Catalog graph, select in *Navigation* the Select Result Template and click `Create a new Select Result Template` to create a new template. +Select in *Graphs* the Corporate Memory Query Catalog graph, select in *Navigation* the Select Result Template and click `Create a new Select Result Template` to create a new template. ![result-template](22-1-2-result-template.png) @@ -108,7 +108,7 @@ END:VCALENDAR As a next step, we will set up the API which serves the data in the format we defined in the previous template. -Select in *Graphs* the CMEM Query Catalog graph, select in *Navigation* the Select Query Endpoint and click "Create a new Select Query Endpoint" to create a new endpoint. +Select in *Graphs* the Corporate Memory Query Catalog graph, select in *Navigation* the Select Query Endpoint and click "Create a new Select Query Endpoint" to create a new endpoint. ![query-endpoint](22-1-3-query-endpoint.png) diff --git a/docs/deploy-and-configure/configuration/access-conditions/index.md b/docs/deploy-and-configure/configuration/access-conditions/index.md index 24a80d1dc..d5452551b 100644 --- a/docs/deploy-and-configure/configuration/access-conditions/index.md +++ b/docs/deploy-and-configure/configuration/access-conditions/index.md @@ -307,7 +307,7 @@ The following list, provides grants which work together: - Allowed Actions: **Explore - Knowledge Graphs Exploration** (`:Explore-KnowledgeGraphs`) - Allow read graphs: - - **CMEM Shape Catalog** (`https://vocab.eccenca.com/shacl/`) + - **CMEM Shapes Catalog** (`https://vocab.eccenca.com/shacl/`) - Any other graph the user should be able to explore - Allow write graphs: - (optional) Any graph the user should be able to change diff --git a/docs/deploy-and-configure/configuration/caveats/index.md b/docs/deploy-and-configure/configuration/caveats/index.md index 322f2b9d2..a8239e4c0 100644 --- a/docs/deploy-and-configure/configuration/caveats/index.md +++ b/docs/deploy-and-configure/configuration/caveats/index.md @@ -25,7 +25,7 @@ See [GraphDB technical-requirements](https://graphdb.ontotext.com/documentation/ ## Load Balancer -Depending on the deployment, load balancers or proxies in between Corporate Memory components (such as Build (DataIntegration) <-> Explore backend, Explore backend <-> Triplestore) tend to drop long running TCP connections when they are idle. +Depending on the deployment, load balancers or proxies in between eccenca Corporate Memory components (such as Build (DataIntegration) <-> Explore backend, Explore backend <-> Triplestore) tend to drop long running TCP connections when they are idle. As e.g. SPARQL Update requests may be idle for a long time (while the update is performed) the client will not receive the response. Hence, DI workflow execution will not proceed even though the update went through. diff --git a/docs/deploy-and-configure/configuration/dataintegration/index.md b/docs/deploy-and-configure/configuration/dataintegration/index.md index a5f7821e1..325383df0 100644 --- a/docs/deploy-and-configure/configuration/dataintegration/index.md +++ b/docs/deploy-and-configure/configuration/dataintegration/index.md @@ -115,7 +115,7 @@ The following sections describe the available workspace provider plugins and how ### RDF-store Workspace - backend -When running in Corporate Memory, by default the workspace is held in the RDF store configured in the eccenca Explore. +When running in eccenca Corporate Memory, by default the workspace is held in the RDF store configured in the eccenca Explore. The workspace is held using the eccenca Explore backend (DataPlatform), i.e., it requires the `eccencaDataPlatform.url` parameter to be configured. @@ -911,7 +911,7 @@ There exist a number of option for secure JDBC connections via Thrift and Hive: - User impersonation - Server and Client Certificates -Eccenca provides a custom Authentification provider which allows to set 1 user/password combination for JDBC connections via: +eccenca provides a custom Authentification provider which allows to set 1 user/password combination for JDBC connections via: ```code spark.sql.options = { diff --git a/docs/deploy-and-configure/configuration/dataintegration/jdbc/index.md b/docs/deploy-and-configure/configuration/dataintegration/jdbc/index.md index 2106b99cb..30ae8f584 100644 --- a/docs/deploy-and-configure/configuration/dataintegration/jdbc/index.md +++ b/docs/deploy-and-configure/configuration/dataintegration/jdbc/index.md @@ -5,7 +5,7 @@ tags: --- # Setup and use of JDBC Drivers -Corporate Memory supports JDBC connections to database management systems (DBMSs). +eccenca Corporate Memory supports JDBC connections to database management systems (DBMSs). The platform includes several JDBC drivers by default. You can also add and use custom drivers. diff --git a/docs/deploy-and-configure/configuration/docker-orchestration/index.md b/docs/deploy-and-configure/configuration/docker-orchestration/index.md index 4ba34ec27..680012914 100644 --- a/docs/deploy-and-configure/configuration/docker-orchestration/index.md +++ b/docs/deploy-and-configure/configuration/docker-orchestration/index.md @@ -15,7 +15,7 @@ In this document we provide an overview on how the environment files are loaded, ## Environment Files: Loading Sequence The environment files are supplied in the CONFIGFILE environment variable to the make targets inside the orchestration. -For example, in [Scenario: Single Node Cloud Installation](../../installation/scenario-single-node-cloud-installation/index.md) we have created a `prod.env` environment file and created the Corporate Memory instance using `prod.env` configuration: +For example, in [Scenario: Single Node Cloud Installation](../../installation/scenario-single-node-cloud-installation/index.md) we have created a `prod.env` environment file and created the eccenca Corporate Memory instance using `prod.env` configuration: ``` shell-session CONFIGFILE=environments/prod.env make clean-pull-start-bootstrap diff --git a/docs/deploy-and-configure/configuration/explore/dataplatform/application-full.md b/docs/deploy-and-configure/configuration/explore/dataplatform/application-full.md index fa5a328cb..143d040fa 100644 --- a/docs/deploy-and-configure/configuration/explore/dataplatform/application-full.md +++ b/docs/deploy-and-configure/configuration/explore/dataplatform/application-full.md @@ -313,19 +313,19 @@ Use this property to enable or disable the AI capabilities of Explore Changeable configuration settings of the companion mcp server and mcp client integration. Explore can integrate external mcp servers via plain spring ai mcp client configuration -and CMEM MCP servers in the authentication realm of the companion via spring.ai.mcp.client.cmem properties +and eccenca Corporate Memory MCP servers in the authentication realm of the companion via spring.ai.mcp.client.cmem properties Every connection is registered under a name and the tools of a connection are exposed to the LLM prefixed with that name, so that equally named tools of different servers stay distinguishable: external mcp servers: external__ - CMEM MCP servers: cmem__ + Corporate Memory MCP servers: cmem__ Since the complete tool name must not exceed 64 characters, keep the connection names short. ***Property: spring.ai.mcp.client.cmem*** -CMEM MCP Servers requiring bearer token authentication in same realm are configured here +Corporate Memory MCP Servers requiring bearer token authentication in same realm are configured here Both SSE and streamable connections are supported. -The configurations under CMEM are the same as Spring AI mcp client configuration. +The configurations under Corporate Memory are the same as Spring AI mcp client configuration. The name of a connection ("build" in the examples below) is part of the tool names. For instance for Streamable: spring.ai.mcp.client.cmem.streamableConnections.build.url= diff --git a/docs/deploy-and-configure/configuration/explore/dataplatform/application-graphdb-full.md b/docs/deploy-and-configure/configuration/explore/dataplatform/application-graphdb-full.md index fc01381a6..9fcde40c1 100644 --- a/docs/deploy-and-configure/configuration/explore/dataplatform/application-graphdb-full.md +++ b/docs/deploy-and-configure/configuration/explore/dataplatform/application-graphdb-full.md @@ -8,7 +8,7 @@ tags: Configuration example: This example configures a connection with HTTPS to a remote graphdb store () using the workbench import directory -which is shared with the GraphDB instance. The repository will be created on startup of CMEM. +which is shared with the GraphDB instance. The repository will be created on startup of eccenca Corporate Memory. ```yaml store: diff --git a/docs/deploy-and-configure/configuration/graphinsights/index.md b/docs/deploy-and-configure/configuration/graphinsights/index.md index 8a4af03d7..f1cc76980 100644 --- a/docs/deploy-and-configure/configuration/graphinsights/index.md +++ b/docs/deploy-and-configure/configuration/graphinsights/index.md @@ -12,7 +12,7 @@ Follow the instructions related to your deployment type: a) docker-compose or b) ### (a) Enable in docker-compose deployment -In our Corporate Memory docker-orchestration all main configurations can be directed by setting environment +In our eccenca Corporate Memory docker-orchestration all main configurations can be directed by setting environment variables in `environments/config.env`. You can find the environment variables set as usual in `environments/default.env` and `environments/config.env`. @@ -328,7 +328,7 @@ accordingly: ``` yaml global: # GraphInsights service account OAuth credentials (if using GraphInsights). - # Can use the same credentials as CMEM or separate ones. + # Can use the same credentials as Corporate Memory or separate ones. graphinsightsClientId: graph-insights-service-account graphinsightsClientSecret: changeme # IMPORTANT: Change this to a secure value! diff --git a/docs/deploy-and-configure/configuration/keycloak/change-passwords-and-keys/index.md b/docs/deploy-and-configure/configuration/keycloak/change-passwords-and-keys/index.md index 498504306..978007115 100644 --- a/docs/deploy-and-configure/configuration/keycloak/change-passwords-and-keys/index.md +++ b/docs/deploy-and-configure/configuration/keycloak/change-passwords-and-keys/index.md @@ -34,7 +34,7 @@ Set a new password. ## Change credentials of cmem-service-account -Make sure the realm Cmem is selected, go to Clients in left sidebar and edit cmem-service-account: +Make sure the realm `cmem` is selected, go to Clients in left sidebar and edit the `cmem-service-account`: ![cmem-selection](22-1-cmem-selection.png) @@ -44,9 +44,9 @@ Switch to "Credentials" tab and press "Regenerate Secret" Button. ## Change credentials of user accounts -In default configuration, there are two users: user and admin. Both are configured with different groups to have different permissions inside Corporate Memory. +In default configuration, there are two users: user and admin. Both are configured with different groups to have different permissions inside eccenca Corporate Memory. -To change the default passwords, select the Cmem Realm and open Users in the left sidebar: +To change the default passwords, select the `cmem` realm and open Users in the left sidebar: ![users-lookup](22-1-users-lookup.png) diff --git a/docs/deploy-and-configure/configuration/keycloak/index.md b/docs/deploy-and-configure/configuration/keycloak/index.md index 090d7c28e..dbf8f452b 100644 --- a/docs/deploy-and-configure/configuration/keycloak/index.md +++ b/docs/deploy-and-configure/configuration/keycloak/index.md @@ -8,7 +8,7 @@ tags: ## Introduction -This page describes important steps in order to configure Keycloak as an authentication backend for Corporate Memory. +This page describes important steps in order to configure Keycloak as an authentication backend for eccenca Corporate Memory. The screenshots displayed in this documentation were taken from Keycloak v20 using the `keycloak.v2` theme. !!! info diff --git a/docs/deploy-and-configure/configuration/keycloak/using-external-keycloak/index.md b/docs/deploy-and-configure/configuration/keycloak/using-external-keycloak/index.md index 9dc583584..32a5040e3 100644 --- a/docs/deploy-and-configure/configuration/keycloak/using-external-keycloak/index.md +++ b/docs/deploy-and-configure/configuration/keycloak/using-external-keycloak/index.md @@ -4,11 +4,11 @@ tags: - Security - Keycloak --- -# Configure Corporate Memory with an external Keycloak +# Configure eccenca Corporate Memory with an external Keycloak ## Introduction -Maybe you already operate a central Keycloak deployment in your infrastructure or you want to deploy multiple stages of Corporate Memory with a single Keycloak. +Maybe you already operate a central Keycloak deployment in your infrastructure or you want to deploy multiple stages of eccenca Corporate Memory with a single Keycloak. Very often this results in a Keycloak which is deployed in a different domain than your Corporate Memory (such as Corporate Memory is available on `https://cmem.example.com` and Keycloak is available on `https://keycloak.example.com`). For this scenario, this page provides additional configuration requirements. @@ -23,7 +23,7 @@ Depending on your infrastructure around Corporate Memory, you need to change som - Headers for Corporate Memory URLs: - `Access-Control-Allow-Origin: *` -For example, if you are using our helm charts, adapt the followin ingress annotations: +For example, if you are using our helm charts, adapt the following ingress annotations: ``` yaml # KEYCLOAK ingress diff --git a/docs/deploy-and-configure/configuration/production-ready-settings/index.md b/docs/deploy-and-configure/configuration/production-ready-settings/index.md index 4c9fe693c..1d6be7af2 100644 --- a/docs/deploy-and-configure/configuration/production-ready-settings/index.md +++ b/docs/deploy-and-configure/configuration/production-ready-settings/index.md @@ -5,7 +5,7 @@ tags: --- # Production-Ready Settings -If you plan to deploy Corporate Memory in a non-trusted environment, you need to take care about some final configuration steps. +If you plan to deploy eccenca Corporate Memory in a non-trusted environment, you need to take care about some final configuration steps. ## Restrict Redirect URLs diff --git a/docs/deploy-and-configure/configuration/reverse-proxy/index.md b/docs/deploy-and-configure/configuration/reverse-proxy/index.md index 13b040b64..a95f72148 100644 --- a/docs/deploy-and-configure/configuration/reverse-proxy/index.md +++ b/docs/deploy-and-configure/configuration/reverse-proxy/index.md @@ -4,7 +4,7 @@ tags: --- # Reverse Proxy -A reverse proxy forwards all requests from the users to the called service and returns the result to the users. Corporate Memory is tested with an Apache HTTP server as a reverse proxy. +A reverse proxy forwards all requests from the users to the called service and returns the result to the users. eccenca Corporate Memory is tested with an Apache HTTP server as a reverse proxy. Reverse proxy is a necessary component in the Corporate Memory deployment. It enables you to: diff --git a/docs/deploy-and-configure/installation/scenario-k8s-deployment/index.md b/docs/deploy-and-configure/installation/scenario-k8s-deployment/index.md index bab58bf56..6800015a5 100644 --- a/docs/deploy-and-configure/installation/scenario-k8s-deployment/index.md +++ b/docs/deploy-and-configure/installation/scenario-k8s-deployment/index.md @@ -5,7 +5,7 @@ icon: simple/kubernetes ## Introduction -This page describes the basic principles of installing Corporate Memory in a +This page describes the basic principles of installing eccenca Corporate Memory in a Kubernetes cluster with `helm` and `kubectl`. It will not explain the basic principles of Kubernetes or help in installing the required tools. diff --git a/docs/deploy-and-configure/installation/scenario-local-installation/index.md b/docs/deploy-and-configure/installation/scenario-local-installation/index.md index e8e8cbf78..6959a9fea 100644 --- a/docs/deploy-and-configure/installation/scenario-local-installation/index.md +++ b/docs/deploy-and-configure/installation/scenario-local-installation/index.md @@ -33,7 +33,7 @@ Install the requirements gnupg lsb-release gettext zip unzip git \ make=4.3-4.1 vim jq ``` - To install Corporate Memory, you need to modify your local hosts file (located in /etc/hosts), minimal configuration is as follows: + To install eccenca Corporate Memory, you need to modify your local hosts file (located in /etc/hosts), minimal configuration is as follows: ```shell ## diff --git a/docs/deploy-and-configure/installation/scenario-single-node-cloud-installation/index.md b/docs/deploy-and-configure/installation/scenario-single-node-cloud-installation/index.md index c409a4737..e0c020c5b 100644 --- a/docs/deploy-and-configure/installation/scenario-single-node-cloud-installation/index.md +++ b/docs/deploy-and-configure/installation/scenario-single-node-cloud-installation/index.md @@ -53,7 +53,7 @@ sudo apt-get install docker-ce docker-ce-cli containerd.io \ ## Installation -Corporate Memory uses Ontotext GraphDB triple store as default backend. +eccenca Corporate Memory uses Ontotext GraphDB triple store as default backend. Graphdb is available as free version and does not requires a license. You need to copy your license for Ontotext GraphDB to the `license` folder inside Corporate Memory's root folder. @@ -152,7 +152,7 @@ make clean-pull-start-bootstrap make tutorials-import ``` -Optional: you can install cmem as a systemd service for this use these commands as root or sudo: +Optional: you can install Corporate Memory as a systemd service for this use these commands as root or sudo: ```bash cp /opt/cmem-orchestration/conf/systemd/cmem-orchestration.service \ diff --git a/docs/develop/accessing-graphs-with-java-applications/index.md b/docs/develop/accessing-graphs-with-java-applications/index.md index 035f6c4a7..83dd35d5f 100644 --- a/docs/develop/accessing-graphs-with-java-applications/index.md +++ b/docs/develop/accessing-graphs-with-java-applications/index.md @@ -9,7 +9,7 @@ hide: ## Introduction -This short recipe covers how to connect to Corporate Memory using a Java program. +This short recipe covers how to connect to eccenca Corporate Memory using a Java program. Such program can connect to Corporate Memory at any time autonomously, independently of whether a user is logged in or not. ## Java Example diff --git a/docs/develop/cmem-client-api/api-reference/auth_provider/client_credentials/index.md b/docs/develop/cmem-client-api/api-reference/auth_provider/client_credentials/index.md index 76f3a2aa8..c63c7cc2a 100644 --- a/docs/develop/cmem-client-api/api-reference/auth_provider/client_credentials/index.md +++ b/docs/develop/cmem-client-api/api-reference/auth_provider/client_credentials/index.md @@ -18,7 +18,7 @@ accessing eccenca Corporate Memory via OAuth 2.0. This flow is designed for machine-to-machine authentication where no user interaction is required. The Client Credentials Flow exchanges client ID and client secret for an access -token directly with the authorization server. It's ideal for backend services, +token directly with the authorization server. It is ideal for backend services, APIs, and automated systems that need to authenticate without user involvement. This implementation handles token caching and automatic renewal when tokens expire. @@ -46,7 +46,7 @@ authentication with Corporate Memory via Keycloak. This flow exchanges client cr (client ID and secret) directly for access tokens without user interaction. The provider handles automatic token caching and refresh, ensuring that get_access_token() -always returns a valid, non-expired token. It's designed for backend services, CLIs, +always returns a valid, non-expired token. It is designed for backend services, CLIs, daemons, and other automated systems that need to authenticate as an application rather than on behalf of a user. @@ -233,7 +233,7 @@ from_env(config) Create a Client Credentials Flow provider from environment variables. This factory method creates a provider instance by reading OAuth client -credentials from environment variables. It's the recommended way to +credentials from environment variables. It is the recommended way to create providers in production environments where credentials are managed externally. diff --git a/docs/develop/cmem-client-api/api-reference/auth_provider/password/index.md b/docs/develop/cmem-client-api/api-reference/auth_provider/password/index.md index 72eb42d23..7df2fe4af 100644 --- a/docs/develop/cmem-client-api/api-reference/auth_provider/password/index.md +++ b/docs/develop/cmem-client-api/api-reference/auth_provider/password/index.md @@ -18,7 +18,7 @@ which allows highly-trusted applications to authenticate users by collecting their username and password credentials directly. Security Warning: This flow should only be used by absolutely trusted -applications as it requires handling user passwords directly. It's typically +applications as it requires handling user passwords directly. It is typically used for legacy applications or first-party applications where other OAuth flows are not feasible. @@ -48,7 +48,7 @@ credentials (username and password) directly for access tokens, bypassing the standard OAuth 2.0 authorization code flow. This provider handles automatic token caching and refresh, ensuring that -get_access_token() always returns a valid token. It's typically used for +get_access_token() always returns a valid token. It is typically used for legacy applications, first-party applications, or scenarios where the standard OAuth flows are not feasible. diff --git a/docs/develop/cmem-client-api/api-reference/auth_provider/prefetched_token/index.md b/docs/develop/cmem-client-api/api-reference/auth_provider/prefetched_token/index.md index bf31bf498..f35b95454 100644 --- a/docs/develop/cmem-client-api/api-reference/auth_provider/prefetched_token/index.md +++ b/docs/develop/cmem-client-api/api-reference/auth_provider/prefetched_token/index.md @@ -19,7 +19,7 @@ for environments where tokens are managed by external systems, CI/CD pipelines, or when integrating with existing authentication infrastructure. The PrefetchedToken provider simply stores and returns a pre-obtained access token -without performing any token refresh or validation. It's the responsibility of the +without performing any token refresh or validation. It is the responsibility of the external system to ensure the token is valid and renewed when necessary. This approach is often used in containerized environments, serverless functions, @@ -50,7 +50,7 @@ This approach is commonly used in: - CI/CD pipelines with token management systems - Serverless functions with external authentication services - Integration with existing authentication infrastructure -- Short-lived execution contexts where token refresh isn't needed +- Short-lived execution contexts where token refresh is not needed **Attributes:** diff --git a/docs/develop/cmem-client-api/api-reference/client/index.md b/docs/develop/cmem-client-api/api-reference/client/index.md index 0f6ec74e8..241ceafbf 100644 --- a/docs/develop/cmem-client-api/api-reference/client/index.md +++ b/docs/develop/cmem-client-api/api-reference/client/index.md @@ -99,8 +99,8 @@ resource allocation. - **_headers** (dict) – Class-level dictionary of HTTP headers shared across instances. - **_auth** ([AuthProvider](../auth_provider/abc/index.md#cmem_client.auth_provider.abc.AuthProvider)) – Authentication provider for obtaining access tokens. - **_http** (Client) – HTTP client instance for making API requests. -- **_workspace** ([BuildWorkspace](../components/workspace/index.md#cmem_client.components.workspace.BuildWorkspace)) – DataIntegration workspace component for build operations. -- **_store** ([GraphStore](../components/graph_store/index.md#cmem_client.components.graph_store.GraphStore)) – DataPlatform graph store component for explore operations. +- **_build** (Build) – Build (DataIntegration) component for workspace operations. +- **_explore** (Explore) – Explore (DataPlatform) component for graph store operations. **Functions:** @@ -162,6 +162,31 @@ configuration or set the auth property manually.
+### `build` {#cmem_client.client.Client.build} + +```python +build: Build +``` + +Get the Build (DataIntegration) component. + +Returns the Build component for managing Corporate Memory's DataIntegration +service. Its `workspace` feature holds the workspace-level operations: import +and export, reload, access control configuration and status. + +**Returns:** + +- Build – The Build component instance, created lazily on first access. + +**Examples:** + +```pycon +>>> from pathlib import Path +>>> client = Client.from_env() +>>> client.build.workspace.import_from_zip(Path("backup.zip")) +>>> client.build.workspace.export_to_zip(Path("new_backup.zip")) +``` + ### `client_accounts` {#cmem_client.client.Client.client_accounts} ```python @@ -288,6 +313,30 @@ information across all Corporate Memory services. >>> print(status.explore.version, status.health) ``` +### `explore` {#cmem_client.client.Client.explore} + +```python +explore: Explore +``` + +Get the Explore (DataPlatform) component. + +Returns the Explore component for managing Corporate Memory's +DataPlatform graph store, including SPARQL queries and updates, bootstrap +data management, and store-level backup/restore functionality. + +**Returns:** + +- Explore – The Explore component instance, created lazily on first access. + +**Examples:** + +```pycon +>>> client = Client.from_env() +>>> store_info = client.explore.self_information +>>> print(f"Store type: {store_info.type}, version: {store_info.version}") +``` + ### `files` {#cmem_client.client.Client.files} ```python @@ -314,7 +363,7 @@ from_context(context, logger=None) Create a client instance configured from a cmem-plugin-base context object. -This method is intended for use inside corporate memory python plugins. +This method is intended for use inside Corporate Memory Python plugins. It extracts connection URLs from the ``SystemContext`` and uses the token provided by the ``UserContext`` for authentication, so no environment variables or credentials need to be supplied manually. @@ -396,7 +445,7 @@ Create a client instance configured from environment variables. This factory method creates a fully configured client by reading configuration and authentication settings from environment variables. -It's the recommended way to create clients in most applications. +This is the recommended way to create clients in most applications. **Parameters:** @@ -415,7 +464,7 @@ It's the recommended way to create clients in most applications. ```pycon >>> my_client = Client.from_env() # Uses CMEM_BASE_URI, OAUTH_* vars ->>> store_info = my_client.store.self_information +>>> store_info = my_client.explore.self_information ``` ### `get_new_httpx_client` {#cmem_client.client.Client.get_new_httpx_client} @@ -455,12 +504,43 @@ Returns: The graph imports repository instance, created lazily on first access. ### `graph_insights` {#cmem_client.client.Client.graph_insights} ```python -graph_insights: GraphInsightsRepository +graph_insights: GraphSnapshotsRepository ``` -Get the Graph Insights repository for managing semspect snapshots. +Deprecated: use [`graph_snapshots`](#cmem_client.client.Client.graph_snapshots) instead. + +Returns the snapshot repository, as before the Graph Insights component became +[`graphinsights`](#cmem_client.client.Client.graphinsights), and raises a +`DeprecationWarning`. Removed in cmem-client 2.0.0. + +**Returns:** + +- [GraphSnapshotsRepository](../repositories/graph_snapshots/index.md#cmem_client.repositories.graph_snapshots.GraphSnapshotsRepository) – The GraphSnapshotsRepository instance. + +### `graph_snapshots` {#cmem_client.client.Client.graph_snapshots} + +```python +graph_snapshots: GraphSnapshotsRepository +``` -Returns: The GraphInsightsRepository instance, created lazily on first access. +Get the repository of the Graph Insights snapshots. + +Returns: The GraphSnapshotsRepository instance, created lazily on first access. + +### `graphinsights` {#cmem_client.client.Client.graphinsights} + +```python +graphinsights: GraphInsights +``` + +Get the Graph Insights component. + +Graph Insights is an optional extension. Its snapshots are managed through +[`graph_snapshots`](#cmem_client.client.Client.graph_snapshots). + +**Returns:** + +- GraphInsights – The GraphInsights component instance, created lazily on first access. ### `graphs` {#cmem_client.client.Client.graphs} @@ -531,7 +611,7 @@ Returns the Marketplace component. **Returns:** -- [Marketplace](../components/marketplace/index.md#cmem_client.components.marketplace.Marketplace) – The Marketplace component instance, created lazily on first access. +- Marketplace – The Marketplace component instance, created lazily on first access. ### `marketplace_packages` {#cmem_client.client.Client.marketplace_packages} @@ -616,6 +696,31 @@ graphs and described using SHACL UI vocabulary. >>> my_query = queries.get(":myQueryId") ``` +### `rule_blocks` {#cmem_client.client.Client.rule_blocks} + +```python +rule_blocks: RuleBlocksRepository +``` + +Get the DataIntegration (build) rule blocks repository. + +Returns the RuleBlocksRepository for managing reusable transformation +rule blocks within projects. Rule blocks require Corporate Memory 26.2 +or newer. + +**Returns:** + +- [RuleBlocksRepository](../repositories/rule_blocks/index.md#cmem_client.repositories.rule_blocks.RuleBlocksRepository) – The RuleBlocksRepository instance, created lazily on first access. + +**Examples:** + +```pycon +>>> client = Client.from_env() +>>> client.rule_blocks.fetch_data() +>>> for rule_block in client.rule_blocks.values(): +... print(rule_block.get_id(), len(rule_block.data.rule_block_model.ports)) +``` + ### `schedulers` {#cmem_client.client.Client.schedulers} ```python @@ -643,26 +748,17 @@ intervals and are identified by a 'project_id:scheduler_id' composite key. ### `store` {#cmem_client.client.Client.store} ```python -store: GraphStore +store: Explore ``` -Get the DataPlatform (explore) graph store component. +Deprecated: use [`explore`](#cmem_client.client.Client.explore) instead. -Returns the GraphStore component for managing Corporate Memory's -DataPlatform graph store, including RDF graph operations, bootstrap -data management, and store-level backup/restore functionality. +Returns the same component as `explore`, and raises a `DeprecationWarning`. +Removed in cmem-client 2.0.0. **Returns:** -- [GraphStore](../components/graph_store/index.md#cmem_client.components.graph_store.GraphStore) – The GraphStore component instance, created lazily on first access. - -**Examples:** - -```pycon ->>> client = Client.from_env() ->>> store_info = client.store.self_information ->>> print(f"Store type: {store_info.type}, version: {store_info.version}") -``` +- Explore – The Explore component instance. ### `user_accounts` {#cmem_client.client.Client.user_accounts} @@ -760,24 +856,14 @@ Returns: The workflows repository instance, created lazily on first access. workspace: BuildWorkspace ``` -Get the DataIntegration (build) workspace component. +Deprecated: use `build.workspace` of [`build`](#cmem_client.client.Client.build) instead. -Returns the BuildWorkspace component for managing Corporate Memory's -DataIntegration workspace, including projects, datasets, transformations, -and workspace-level import/export operations. +Returns the same component as `build.workspace`, and raises a `DeprecationWarning`. +Removed in cmem-client 2.0.0. **Returns:** -- [BuildWorkspace](../components/workspace/index.md#cmem_client.components.workspace.BuildWorkspace) – The BuildWorkspace component instance, created lazily on first access. - -**Examples:** - -```pycon ->>> from pathlib import Path ->>> client = Client.from_env() ->>> client.workspace.import_from_zip(Path("backup.zip")) ->>> client.workspace.export_to_zip(Path("new_backup.zip")) -``` +- BuildWorkspace – The BuildWorkspace component instance. ### `workspace_configs` {#cmem_client.client.Client.workspace_configs} diff --git a/docs/develop/cmem-client-api/api-reference/components/.pages b/docs/develop/cmem-client-api/api-reference/components/.pages index 2c8383fd4..5c7647c01 100644 --- a/docs/develop/cmem-client-api/api-reference/components/.pages +++ b/docs/develop/cmem-client-api/api-reference/components/.pages @@ -2,7 +2,7 @@ title: components nav: - deployment: deployment - - graph_store: graph_store + - build: build + - explore: explore + - graphinsights: graphinsights - marketplace: marketplace - - sparql_wrapper: sparql_wrapper - - workspace: workspace diff --git a/docs/develop/cmem-client-api/api-reference/components/build/.pages b/docs/develop/cmem-client-api/api-reference/components/build/.pages new file mode 100644 index 000000000..474be5e0d --- /dev/null +++ b/docs/develop/cmem-client-api/api-reference/components/build/.pages @@ -0,0 +1,6 @@ +--- +title: build +nav: + - component: component + - logs: logs + - workspace: workspace diff --git a/docs/develop/cmem-client-api/api-reference/components/build/component/index.md b/docs/develop/cmem-client-api/api-reference/components/build/component/index.md new file mode 100644 index 000000000..aa3646d91 --- /dev/null +++ b/docs/develop/cmem-client-api/api-reference/components/build/component/index.md @@ -0,0 +1,83 @@ +--- +title: "cmem-client: components.build.component module" +description: "Corporate Memory Build (DataIntegration) component." +tags: + - API + - Python + - cmem-client +--- + +# `cmem_client.components.build.component` {#cmem_client.components.build.component} + + + +Corporate Memory Build (DataIntegration) component. + +The Build component bundles the operations of the Build service which are not +collections of items: the workspace feature (import and export of the whole workspace, +reload, access control configuration, marshalling plugins and the loading status) and +the log buffer feature. Each feature lives in its own module of this package. + +**Classes:** + +- [**Build**](#cmem_client.components.build.component.Build) – High-level interface for the Corporate Memory Build (DataIntegration) service. + +## `Build` {#cmem_client.components.build.component.Build} + +```python +Build(client) +``` + +High-level interface for the Corporate Memory Build (DataIntegration) service. + +**Attributes:** + +- [**workspace**](#cmem_client.components.build.component.Build.workspace) ([BuildWorkspace](../../../components/build/workspace/index.md#cmem_client.components.build.workspace.BuildWorkspace)) – The workspace feature, for operations on the whole DataIntegration workspace. +- [**logs**](#cmem_client.components.build.component.Build.logs) ([BuildLogs](../../../components/build/logs/index.md#cmem_client.components.build.logs.BuildLogs)) – The log buffer feature, for the most recent log lines of Build and the state of its buffer. + +**Functions:** + +- [**get_health**](#cmem_client.components.build.component.Build.get_health) – Report version and health of the Build (DataIntegration) service. + +**Parameters:** + +- **client** ([Client](../../../client/index.md#cmem_client.client.Client)) – A configured Corporate Memory client instance with +authentication and endpoint configuration. + +### `get_health` {#cmem_client.components.build.component.Build.get_health} + +```python +get_health() +``` + +Report version and health of the Build (DataIntegration) service. + +Reads the service's `/version` and `/health`. + +**Returns:** + +- [ComponentStatus](../../../models/status/index.md#cmem_client.models.status.ComponentStatus) – The status of Build. An unreachable or failing service is reported DOWN with +- [ComponentStatus](../../../models/status/index.md#cmem_client.models.status.ComponentStatus) – the error, instead of raising. + +### `logger` {#cmem_client.components.build.component.Build.logger} + +```python +logger = logging.getLogger(f'{self._client.logger.name}.{self.__class__.__name__}') +``` + +### `logs` {#cmem_client.components.build.component.Build.logs} + +```python +logs: BuildLogs = BuildLogs(client=client) +``` + +The in-memory log buffer of Build: the most recent lines and the state of the buffer. + +### `workspace` {#cmem_client.components.build.component.Build.workspace} + +```python +workspace: BuildWorkspace = BuildWorkspace(client=client) +``` + +Operations on the whole DataIntegration workspace: import, export, reload and status. + diff --git a/docs/develop/cmem-client-api/api-reference/components/build/logs/index.md b/docs/develop/cmem-client-api/api-reference/components/build/logs/index.md new file mode 100644 index 000000000..be896e172 --- /dev/null +++ b/docs/develop/cmem-client-api/api-reference/components/build/logs/index.md @@ -0,0 +1,115 @@ +--- +title: "cmem-client: components.build.logs module" +description: "The in-memory log buffer of Build (DataIntegration)." +tags: + - API + - Python + - cmem-client +--- + +# `cmem_client.components.build.logs` {#cmem_client.components.build.logs} + + + +The in-memory log buffer of Build (DataIntegration). + +**Classes:** + +- [**BuildLogs**](#cmem_client.components.build.logs.BuildLogs) – Read the in-memory log buffer of Build: the most recent lines, and its state. + +## `BuildLogs` {#cmem_client.components.build.logs.BuildLogs} + +Bases: _LogBuffer + +Read the in-memory log buffer of Build: the most recent lines, and its state. + +Build offers the same log API as Explore, with one difference: it does not record +request ids, so its lines cannot be filtered by one. Build would ignore such a filter +and answer with every line it holds, which is why `get_tail()` does not offer it. + +**Examples:** + +```pycon +>>> from cmem_client.client import Client +>>> client = Client.from_env() +>>> tail = client.build.logs.get_tail(level="WARN", limit=10) +>>> [line.message for line in tail.lines] +``` + +**Functions:** + +- [**get_status**](#cmem_client.components.build.logs.BuildLogs.get_status) – Read the state of the in-memory log buffer of Build. +- [**get_tail**](#cmem_client.components.build.logs.BuildLogs.get_tail) – Read the most recent log lines of Build. + +**Attributes:** + +- [**logger**](#cmem_client.components.build.logs.BuildLogs.logger) – + +### `get_status` {#cmem_client.components.build.logs.BuildLogs.get_status} + +```python +get_status() +``` + +Read the state of the in-memory log buffer of Build. + +Reports whether lines are captured at all, at which level, how many are held and +which loggers are excluded. This is the way to find out why a log read returns +fewer lines than expected, or none. + +**Returns:** + +- [LogBufferStatus](../../../models/logs/index.md#cmem_client.models.logs.LogBufferStatus) – The state and configuration of the buffer of the instance which answered. + +**Raises:** + +- HTTPStatusError – If the request fails, for example because the deployment +does not know the endpoint or the user is not authorized for it. +- LogsReadError – If the answer is not a log buffer status. + +### `get_tail` {#cmem_client.components.build.logs.BuildLogs.get_tail} + +```python +get_tail(since=None, limit=None, level=None, logger=None, contains=None) +``` + +Read the most recent log lines of Build. + +The lines are returned oldest first, and every filter is applied by Build before +the limit. Which end the limit cuts depends on `since`: without it Build answers +the newest `limit` matching lines, and with it the oldest `limit` lines after the +cursor, so a follower pages forward instead of skipping ahead. + +To follow the log, pass the `last_sequence` of the previous response as `since` +of the next request. A `dropped` above zero in a response means the client fell +behind and those lines are no longer buffered. Sequence numbers belong to the +instance which answered, so a changed `instance_id` invalidates the cursor. + +**Parameters:** + +- **since** (int | None) – Return only lines with a sequence higher than this one. +- **limit** (int | None) – Maximum number of lines to return. Build clamps this to its own +configured maximum rather than rejecting a larger value. Without a limit +Build answers its own default page, which is smaller than its maximum. +- **level** ([LogLevel](../../../models/logs/index.md#cmem_client.models.logs.LogLevel) | str | None) – Minimum level a line has to be logged at. Asking for WARN returns +WARN and ERROR lines. +- **logger** (list[str] | None) – Return only lines of loggers starting with one of these prefixes. +- **contains** (list[str] | None) – Return only lines containing all of these substrings, ignoring +case. + +**Returns:** + +- [LogTailResponse](../../../models/logs/index.md#cmem_client.models.logs.LogTailResponse) – The matching lines together with the state of the read. + +**Raises:** + +- HTTPStatusError – If the request fails, for example because a filter value is +not valid, the buffer is disabled or the user is not authorized. +- LogsReadError – If the answer is not a log buffer response. + +### `logger` {#cmem_client.components.build.logs.BuildLogs.logger} + +```python +logger = logging.getLogger(f'{self._client.logger.name}.{self.__class__.__name__}') +``` + diff --git a/docs/develop/cmem-client-api/api-reference/components/workspace/index.md b/docs/develop/cmem-client-api/api-reference/components/build/workspace/index.md similarity index 76% rename from docs/develop/cmem-client-api/api-reference/components/workspace/index.md rename to docs/develop/cmem-client-api/api-reference/components/build/workspace/index.md index e6b27ecff..206e83e9a 100644 --- a/docs/develop/cmem-client-api/api-reference/components/workspace/index.md +++ b/docs/develop/cmem-client-api/api-reference/components/build/workspace/index.md @@ -1,5 +1,5 @@ --- -title: "cmem-client: components.workspace module" +title: "cmem-client: components.build.workspace module" description: "Corporate Memory DataIntegration (build) workspace management." tags: - API @@ -7,7 +7,7 @@ tags: - cmem-client --- -# `cmem_client.components.workspace` {#cmem_client.components.workspace} +# `cmem_client.components.build.workspace` {#cmem_client.components.build.workspace} @@ -24,9 +24,9 @@ scenarios. **Classes:** -- [**BuildWorkspace**](#cmem_client.components.workspace.BuildWorkspace) – High-level interface for Corporate Memory DataIntegration workspace operations. +- [**BuildWorkspace**](#cmem_client.components.build.workspace.BuildWorkspace) – High-level interface for Corporate Memory DataIntegration workspace operations. -## `BuildWorkspace` {#cmem_client.components.workspace.BuildWorkspace} +## `BuildWorkspace` {#cmem_client.components.build.workspace.BuildWorkspace} ```python BuildWorkspace(client) @@ -51,7 +51,7 @@ a convenient interface for workspace-wide administrative tasks. **Attributes:** -- **_client** ([Client](../../client/index.md#cmem_client.client.Client)) – The Corporate Memory client instance used for API communication. +- **_client** ([Client](../../../client/index.md#cmem_client.client.Client)) – The Corporate Memory client instance used for API communication.
Administrative Operations @@ -73,19 +73,19 @@ which provides CRUD operations for specific DataIntegration projects. **Functions:** -- [**export_to_zip**](#cmem_client.components.workspace.BuildWorkspace.export_to_zip) – Export a complete backup of the workspace as a ZIP archive. -- [**get_marshalling_plugins**](#cmem_client.components.workspace.BuildWorkspace.get_marshalling_plugins) – Get the list of marshalling plugins. -- [**get_status**](#cmem_client.components.workspace.BuildWorkspace.get_status) – Get the loading status of the whole workspace. -- [**import_from_zip**](#cmem_client.components.workspace.BuildWorkspace.import_from_zip) – Import and restore a complete workspace backup from a ZIP archive. -- [**reload_workspace**](#cmem_client.components.workspace.BuildWorkspace.reload_workspace) – Reload the workspace. -- [**retrieve_access_control_configuration**](#cmem_client.components.workspace.BuildWorkspace.retrieve_access_control_configuration) – Retrieves the current access control configuration. +- [**export_to_zip**](#cmem_client.components.build.workspace.BuildWorkspace.export_to_zip) – Export a complete backup of the workspace as a ZIP archive. +- [**get_marshalling_plugins**](#cmem_client.components.build.workspace.BuildWorkspace.get_marshalling_plugins) – Get the list of marshalling plugins. +- [**get_status**](#cmem_client.components.build.workspace.BuildWorkspace.get_status) – Get the loading status of the whole workspace. +- [**import_from_zip**](#cmem_client.components.build.workspace.BuildWorkspace.import_from_zip) – Import and restore a complete workspace backup from a ZIP archive. +- [**reload_workspace**](#cmem_client.components.build.workspace.BuildWorkspace.reload_workspace) – Reload the workspace. +- [**retrieve_access_control_configuration**](#cmem_client.components.build.workspace.BuildWorkspace.retrieve_access_control_configuration) – Retrieves the current access control configuration. Creates a BuildWorkspace component that uses the provided client for API communication with the DataIntegration workspace endpoints. **Parameters:** -- **client** ([Client](../../client/index.md#cmem_client.client.Client)) – A configured Corporate Memory client instance with +- **client** ([Client](../../../client/index.md#cmem_client.client.Client)) – A configured Corporate Memory client instance with authentication and endpoint configuration.
@@ -97,7 +97,7 @@ instantiation is rarely needed in normal usage.
-### `export_to_zip` {#cmem_client.components.workspace.BuildWorkspace.export_to_zip} +### `export_to_zip` {#cmem_client.components.build.workspace.BuildWorkspace.export_to_zip} ```python export_to_zip(path, marshalling_plugin='xmlZip', include_access_conditions=False, export_user_data=True) @@ -137,7 +137,7 @@ timestamps and account names). If False, this data is removed from the archive. errors, or insufficient permissions. - OSError – If the specified path cannot be written to due to file system permissions or disk space issues. -- [RepositoryModificationError](../../exceptions/index.md#cmem_client.exceptions.RepositoryModificationError) – If the server reports that the export failed. +- [RepositoryModificationError](../../../exceptions/index.md#cmem_client.exceptions.RepositoryModificationError) – If the server reports that the export failed.
Performance Notes @@ -180,7 +180,7 @@ Use import_from_zip() to restore workspace archives created by this method.
-### `get_marshalling_plugins` {#cmem_client.components.workspace.BuildWorkspace.get_marshalling_plugins} +### `get_marshalling_plugins` {#cmem_client.components.build.workspace.BuildWorkspace.get_marshalling_plugins} ```python get_marshalling_plugins() @@ -188,7 +188,7 @@ get_marshalling_plugins() Get the list of marshalling plugins. -### `get_status` {#cmem_client.components.workspace.BuildWorkspace.get_status} +### `get_status` {#cmem_client.components.build.workspace.BuildWorkspace.get_status} ```python get_status() @@ -201,9 +201,9 @@ Only projects with at least one failed task are listed. **Returns:** -- [WorkspaceStatus](../../models/workspace_status/index.md#cmem_client.models.workspace_status.WorkspaceStatus) – The workspace status, listing the failed tasks per project. +- [WorkspaceStatus](../../../models/workspace_status/index.md#cmem_client.models.workspace_status.WorkspaceStatus) – The workspace status, listing the failed tasks per project. -### `import_from_zip` {#cmem_client.components.workspace.BuildWorkspace.import_from_zip} +### `import_from_zip` {#cmem_client.components.build.workspace.BuildWorkspace.import_from_zip} ```python import_from_zip(path, marshalling_plugin='xmlZip', include_access_conditions=False) @@ -243,7 +243,7 @@ for detailed import results and any warnings or errors. errors, insufficient permissions, or invalid archive format. - OSError – If the specified backup file cannot be read due to file system permissions or if the file does not exist. -- [RepositoryModificationError](../../exceptions/index.md#cmem_client.exceptions.RepositoryModificationError) – If the server reports that the import failed. +- [RepositoryModificationError](../../../exceptions/index.md#cmem_client.exceptions.RepositoryModificationError) – If the server reports that the import failed.
Important Considerations @@ -281,13 +281,13 @@ Use export_to_zip() to create workspace archives for import with this method.
-### `logger` {#cmem_client.components.workspace.BuildWorkspace.logger} +### `logger` {#cmem_client.components.build.workspace.BuildWorkspace.logger} ```python logger = logging.getLogger(f'{self._client.logger.name}.{self.__class__.__name__}') ``` -### `reload_workspace` {#cmem_client.components.workspace.BuildWorkspace.reload_workspace} +### `reload_workspace` {#cmem_client.components.build.workspace.BuildWorkspace.reload_workspace} ```python reload_workspace() @@ -295,7 +295,7 @@ reload_workspace() Reload the workspace. -### `retrieve_access_control_configuration` {#cmem_client.components.workspace.BuildWorkspace.retrieve_access_control_configuration} +### `retrieve_access_control_configuration` {#cmem_client.components.build.workspace.BuildWorkspace.retrieve_access_control_configuration} ```python retrieve_access_control_configuration() diff --git a/docs/develop/cmem-client-api/api-reference/components/deployment/index.md b/docs/develop/cmem-client-api/api-reference/components/deployment/index.md index febe098e3..122667409 100644 --- a/docs/develop/cmem-client-api/api-reference/components/deployment/index.md +++ b/docs/develop/cmem-client-api/api-reference/components/deployment/index.md @@ -15,12 +15,16 @@ Corporate Memory deployment status component. Provides the Deployment component for aggregating version and health information across all Corporate Memory services (DataIntegration, -DataPlatform, shapes catalog, and graph store). +DataPlatform, shapes catalog, graph store, marketplace, and Graph Insights). **Classes:** - [**Deployment**](#cmem_client.components.deployment.Deployment) – High-level interface for Corporate Memory deployment status. +**Attributes:** + +- [**MARKETPLACE_MIN_BUILD_VERSION**](#cmem_client.components.deployment.MARKETPLACE_MIN_BUILD_VERSION) – Earliest (year, major) build (DataIntegration) version that ships the marketplace. + ## `Deployment` {#cmem_client.components.deployment.Deployment} ```python @@ -30,9 +34,10 @@ Deployment(client) High-level interface for Corporate Memory deployment status. Aggregates version and health information across all Corporate Memory -components (DataIntegration, DataPlatform, shapes catalog, graph store). -Per-component failures are captured rather than raised, so a partial -outage still returns a populated StatusInfo for the healthy components. +components (DataIntegration, DataPlatform, shapes catalog, graph store, +marketplace, Graph Insights), as each component reports it with its own +`get_health()`. Per-component failures are captured rather than raised, so a +partial outage still returns a populated StatusInfo for the healthy components. **Functions:** @@ -50,6 +55,10 @@ get_status() Aggregate version and health information across all components. +Each component reports its own health; this adds the parts which depend on more +than one service: the store, which Explore describes, and whether a failing local +marketplace counts, which depends on the Build version. + **Returns:** - [StatusInfo](../../models/status/index.md#cmem_client.models.status.StatusInfo) – StatusInfo with version, health, and error per component. The @@ -61,3 +70,11 @@ Aggregate version and health information across all components. logger = logging.getLogger(f'{self._client.logger.name}.{self.__class__.__name__}') ``` +## `MARKETPLACE_MIN_BUILD_VERSION` {#cmem_client.components.deployment.MARKETPLACE_MIN_BUILD_VERSION} + +```python +MARKETPLACE_MIN_BUILD_VERSION = (26, 2) +``` + +Earliest (year, major) build (DataIntegration) version that ships the marketplace. + diff --git a/docs/develop/cmem-client-api/api-reference/components/explore/.pages b/docs/develop/cmem-client-api/api-reference/components/explore/.pages new file mode 100644 index 000000000..17b5813be --- /dev/null +++ b/docs/develop/cmem-client-api/api-reference/components/explore/.pages @@ -0,0 +1,7 @@ +--- +title: explore +nav: + - component: component + - logs: logs + - sparql: sparql + - titles: titles diff --git a/docs/develop/cmem-client-api/api-reference/components/graph_store/index.md b/docs/develop/cmem-client-api/api-reference/components/explore/component/index.md similarity index 60% rename from docs/develop/cmem-client-api/api-reference/components/graph_store/index.md rename to docs/develop/cmem-client-api/api-reference/components/explore/component/index.md index be4034297..0be7abcfe 100644 --- a/docs/develop/cmem-client-api/api-reference/components/graph_store/index.md +++ b/docs/develop/cmem-client-api/api-reference/components/explore/component/index.md @@ -1,38 +1,32 @@ --- -title: "cmem-client: components.graph_store module" -description: "Corporate Memory DataPlatform (explore) graph store management." +title: "cmem-client: components.explore.component module" +description: "Corporate Memory Explore (DataPlatform) component." tags: - API - Python - cmem-client --- -# `cmem_client.components.graph_store` {#cmem_client.components.graph_store} +# `cmem_client.components.explore.component` {#cmem_client.components.explore.component} -Corporate Memory DataPlatform (explore) graph store management. +Corporate Memory Explore (DataPlatform) component. -This module provides the GraphStore component for managing Corporate Memory's -DataPlatform graph store. The graph store is the primary repository for RDF -data and knowledge graphs, supporting semantic queries and exploration. - -The GraphStore component provides high-level administrative operations including -bootstrap data management, full store backup and restoration, and system -information retrieval. These operations are essential for store maintenance, -deployment, and operational monitoring. +The Explore component bundles the operations of the Explore service which are not +collections of items: the SPARQL endpoint, administrative status, bootstrap and showcase +data, full backup and restore, and information about the graph store instance. Each of +these lives in its own module of this package; this class only delegates to them. **Classes:** -- [**AdminOperationStatus**](#cmem_client.components.graph_store.AdminOperationStatus) – Current administrative operation reported by GET /api/admin/status. -- [**GraphStore**](#cmem_client.components.graph_store.GraphStore) – High-level interface for Corporate Memory DataPlatform graph store operations. -- [**StoreInformation**](#cmem_client.components.graph_store.StoreInformation) – Information about the graph store instance and its capabilities. +- [**Explore**](#cmem_client.components.explore.component.Explore) – High-level interface for the Corporate Memory Explore (DataPlatform) service. **Attributes:** -- [**AUTHORIZATION_GRAPH_URI**](#cmem_client.components.graph_store.AUTHORIZATION_GRAPH_URI) – The URI of the access conditions graph. Deleting or importing this graph requires an authorization refresh. +- [**AUTHORIZATION_GRAPH_URI**](#cmem_client.components.explore.component.AUTHORIZATION_GRAPH_URI) – The URI of the access conditions graph. Deleting or importing this graph requires an authorization refresh. -## `AUTHORIZATION_GRAPH_URI` {#cmem_client.components.graph_store.AUTHORIZATION_GRAPH_URI} +## `AUTHORIZATION_GRAPH_URI` {#cmem_client.components.explore.component.AUTHORIZATION_GRAPH_URI} ```python AUTHORIZATION_GRAPH_URI = 'https://ns.eccenca.com/data/ac/' @@ -40,79 +34,28 @@ AUTHORIZATION_GRAPH_URI = 'https://ns.eccenca.com/data/ac/' The URI of the access conditions graph. Deleting or importing this graph requires an authorization refresh. -## `AdminOperationStatus` {#cmem_client.components.graph_store.AdminOperationStatus} - -Bases: StrEnum - -Current administrative operation reported by GET /api/admin/status. - -**Attributes:** - -- [**BACKUP**](#cmem_client.components.graph_store.AdminOperationStatus.BACKUP) – -- [**BOOTSTRAP**](#cmem_client.components.graph_store.AdminOperationStatus.BOOTSTRAP) – -- [**DROP_ALL_GRAPHS**](#cmem_client.components.graph_store.AdminOperationStatus.DROP_ALL_GRAPHS) – -- [**NONE**](#cmem_client.components.graph_store.AdminOperationStatus.NONE) – -- [**RESTORE**](#cmem_client.components.graph_store.AdminOperationStatus.RESTORE) – -- [**SHOWCASE**](#cmem_client.components.graph_store.AdminOperationStatus.SHOWCASE) – - -### `BACKUP` {#cmem_client.components.graph_store.AdminOperationStatus.BACKUP} - -```python -BACKUP = 'BACKUP' -``` - -### `BOOTSTRAP` {#cmem_client.components.graph_store.AdminOperationStatus.BOOTSTRAP} - -```python -BOOTSTRAP = 'BOOTSTRAP' -``` - -### `DROP_ALL_GRAPHS` {#cmem_client.components.graph_store.AdminOperationStatus.DROP_ALL_GRAPHS} - -```python -DROP_ALL_GRAPHS = 'DROP_ALL_GRAPHS' -``` - -### `NONE` {#cmem_client.components.graph_store.AdminOperationStatus.NONE} +## `Explore` {#cmem_client.components.explore.component.Explore} ```python -NONE = 'NONE' +Explore(client) ``` -### `RESTORE` {#cmem_client.components.graph_store.AdminOperationStatus.RESTORE} - -```python -RESTORE = 'RESTORE' -``` - -### `SHOWCASE` {#cmem_client.components.graph_store.AdminOperationStatus.SHOWCASE} - -```python -SHOWCASE = 'SHOWCASE' -``` +High-level interface for the Corporate Memory Explore (DataPlatform) service. -## `GraphStore` {#cmem_client.components.graph_store.GraphStore} - -```python -GraphStore(client) -``` - -High-level interface for Corporate Memory DataPlatform graph store operations. - -The GraphStore component provides administrative and operational methods for +The Explore component provides administrative and operational methods for managing the Corporate Memory DataPlatform graph store. It handles store-level operations including bootstrap data management, full backup and restoration, and system information retrieval. This component abstracts the complexities of the DataPlatform API and provides -a convenient interface for common graph store management tasks. It's designed +a convenient interface for common graph store management tasks. It is designed for administrative operations rather than individual graph manipulation (use repositories for graph-level operations). **Attributes:** -- **_client** ([Client](../../client/index.md#cmem_client.client.Client)) – The Corporate Memory client instance used for API communication. -- **_sparql_wrapper** ([SPARQLWrapper](../../components/sparql_wrapper/index.md#cmem_client.components.sparql_wrapper.SPARQLWrapper)) – SPARQLWrapper instance for rdflib SPARQL queries. +- **_client** ([Client](../../../client/index.md#cmem_client.client.Client)) – The Corporate Memory client instance used for API communication. +- **_sparql_wrapper** ([SPARQLWrapper](../../../components/explore/sparql/index.md#cmem_client.components.explore.sparql.SPARQLWrapper)) – SPARQLWrapper instance for rdflib SPARQL queries.
Administrative Operations @@ -132,18 +75,18 @@ which provides CRUD operations for specific RDF graphs. **Functions:** -- [**create_showcase_data**](#cmem_client.components.graph_store.GraphStore.create_showcase_data) – Create showcase data in the graph store. -- [**delete_bootstrap_data**](#cmem_client.components.graph_store.GraphStore.delete_bootstrap_data) – Delete bootstrap data from the graph store. -- [**export_to_zip**](#cmem_client.components.graph_store.GraphStore.export_to_zip) – Export a complete backup of the graph store as a ZIP archive. -- [**import_bootstrap_data**](#cmem_client.components.graph_store.GraphStore.import_bootstrap_data) – Import or update bootstrap data in the graph store. -- [**import_from_zip**](#cmem_client.components.graph_store.GraphStore.import_from_zip) – Import and restore a complete graph store backup from a ZIP archive. - -Creates a GraphStore component that uses the provided client for -API communication with the DataPlatform graph store. +- [**create_showcase_data**](#cmem_client.components.explore.component.Explore.create_showcase_data) – Create showcase data in the graph store. +- [**delete_bootstrap_data**](#cmem_client.components.explore.component.Explore.delete_bootstrap_data) – Delete bootstrap data from the graph store. +- [**export_to_zip**](#cmem_client.components.explore.component.Explore.export_to_zip) – Export a complete backup of the graph store as a ZIP archive. +- [**get_health**](#cmem_client.components.explore.component.Explore.get_health) – Report version and health of the Explore (DataPlatform) service. +- [**get_shapes_health**](#cmem_client.components.explore.component.Explore.get_shapes_health) – Report version and health of the shapes catalog served by Explore. +- [**import_bootstrap_data**](#cmem_client.components.explore.component.Explore.import_bootstrap_data) – Import or update bootstrap data in the graph store. +- [**import_from_zip**](#cmem_client.components.explore.component.Explore.import_from_zip) – Import and restore a complete graph store backup from a ZIP archive. +- [**wait_for_admin_idle**](#cmem_client.components.explore.component.Explore.wait_for_admin_idle) – Poll the administrative status until no administrative operation is in progress. **Parameters:** -- **client** ([Client](../../client/index.md#cmem_client.client.Client)) – A configured Corporate Memory client instance with +- **client** ([Client](../../../client/index.md#cmem_client.client.Client)) – A configured Corporate Memory client instance with authentication and endpoint configuration.
@@ -155,7 +98,7 @@ instantiation is rarely needed in normal usage.
-### `create_showcase_data` {#cmem_client.components.graph_store.GraphStore.create_showcase_data} +### `create_showcase_data` {#cmem_client.components.explore.component.Explore.create_showcase_data} ```python create_showcase_data(scale_factor=None) @@ -179,7 +122,7 @@ A value of 10 results in around 40k triples, a value of - HTTPError – If the showcase creation request fails due to network issues or server errors. -### `delete_bootstrap_data` {#cmem_client.components.graph_store.GraphStore.delete_bootstrap_data} +### `delete_bootstrap_data` {#cmem_client.components.explore.component.Explore.delete_bootstrap_data} ```python delete_bootstrap_data() @@ -217,7 +160,7 @@ operation should typically be followed by import_bootstrap_data().
-### `export_to_zip` {#cmem_client.components.graph_store.GraphStore.export_to_zip} +### `export_to_zip` {#cmem_client.components.explore.component.Explore.export_to_zip} ```python export_to_zip(path) @@ -277,7 +220,39 @@ Use import_from_zip() to restore from backup archives created by this method.
-### `import_bootstrap_data` {#cmem_client.components.graph_store.GraphStore.import_bootstrap_data} +### `get_health` {#cmem_client.components.explore.component.Explore.get_health} + +```python +get_health() +``` + +Report version and health of the Explore (DataPlatform) service. + +Reads the service's `/actuator/info` and `/actuator/health`. Both answers are kept +in the result: `info` also carries the store block (type and version of the graph +store), and is only set when both requests succeed. + +**Returns:** + +- [ExploreStatus](../../../models/status/index.md#cmem_client.models.status.ExploreStatus) – The status of Explore. An unreachable or failing service is reported DOWN with +- [ExploreStatus](../../../models/status/index.md#cmem_client.models.status.ExploreStatus) – the error, instead of raising. + +### `get_shapes_health` {#cmem_client.components.explore.component.Explore.get_shapes_health} + +```python +get_shapes_health() +``` + +Report version and health of the shapes catalog served by Explore. + +Reads the version the shapes catalog graph declares, with a SPARQL query. + +**Returns:** + +- [ComponentStatus](../../../models/status/index.md#cmem_client.models.status.ComponentStatus) – The status of the shapes catalog: UP with its version, DOWN without a version +- [ComponentStatus](../../../models/status/index.md#cmem_client.models.status.ComponentStatus) – or with the error of a failing query, instead of raising. + +### `import_bootstrap_data` {#cmem_client.components.explore.component.Explore.import_bootstrap_data} ```python import_bootstrap_data() @@ -299,7 +274,7 @@ issues or server errors. Note This operation may take some time. -It's typically performed during system initialization or when +It is typically performed during system initialization or when updating to new Corporate Memory versions that include new system vocabularies. @@ -314,7 +289,7 @@ system vocabularies. -### `import_from_zip` {#cmem_client.components.graph_store.GraphStore.import_from_zip} +### `import_from_zip` {#cmem_client.components.explore.component.Explore.import_from_zip} ```python import_from_zip(path) @@ -384,13 +359,21 @@ Use export_to_zip() to create backup archives for import with this method. -### `logger` {#cmem_client.components.graph_store.GraphStore.logger} +### `logger` {#cmem_client.components.explore.component.Explore.logger} ```python logger = logging.getLogger(f'{self._client.logger.name}.{self.__class__.__name__}') ``` -### `self_information` {#cmem_client.components.graph_store.GraphStore.self_information} +### `logs` {#cmem_client.components.explore.component.Explore.logs} + +```python +logs: ExploreLogs = ExploreLogs(client=client) +``` + +The in-memory log buffer of Explore: the most recent lines and the state of the buffer. + +### `self_information` {#cmem_client.components.explore.component.Explore.self_information} ```python self_information: StoreInformation @@ -406,7 +389,7 @@ provides real-time metadata about the running graph store instance. **Returns:** -- **StoreInformation** ([StoreInformation](#cmem_client.components.graph_store.StoreInformation)) – A model containing store type and version information. +- **StoreInformation** ([StoreInformation](../../../models/explore/index.md#cmem_client.models.explore.StoreInformation)) – A model containing store type and version information. The returned object includes the store implementation name (e.g., "GRAPHDB", "TENTRIS") and its version string. @@ -437,7 +420,7 @@ information due to unexpected response format. -### `sparql` {#cmem_client.components.graph_store.GraphStore.sparql} +### `sparql` {#cmem_client.components.explore.component.Explore.sparql} ```python sparql: SPARQLWrapper @@ -452,50 +435,45 @@ authentication handling. **Returns:** -- [SPARQLWrapper](../../components/sparql_wrapper/index.md#cmem_client.components.sparql_wrapper.SPARQLWrapper) – The SPARQLWrapper component instance, created lazily on first access. +- [SPARQLWrapper](../../../components/explore/sparql/index.md#cmem_client.components.explore.sparql.SPARQLWrapper) – The SPARQLWrapper component instance, created lazily on first access. **Examples:** ```pycon >>> from cmem_client.client import Client >>> client = Client.from_env() ->>> sparql_wrapper = client.store.sparql +>>> sparql_wrapper = client.explore.sparql ``` -## `StoreInformation` {#cmem_client.components.graph_store.StoreInformation} +### `titles` {#cmem_client.components.explore.component.Explore.titles} -Bases: [Model](../../models/base/index.md#cmem_client.models.base.Model) - -Information about the graph store instance and its capabilities. - -This model represents metadata about the DataPlatform graph store, -including the store type and version information. This information -is useful for compatibility checks, monitoring, and debugging. - -**Attributes:** +```python +titles: ExploreTitles = ExploreTitles(client=client) +``` -- [**type**](#cmem_client.components.graph_store.StoreInformation.type) (str) – The type of graph store (e.g., "GRAPHDB", "TENTRIS"). -- [**version**](#cmem_client.components.graph_store.StoreInformation.version) (str) – The version string of the graph store implementation. +Human-readable titles of IRIs, resolved in batches and cached. -### `model_config` {#cmem_client.components.graph_store.StoreInformation.model_config} +### `wait_for_admin_idle` {#cmem_client.components.explore.component.Explore.wait_for_admin_idle} ```python -model_config = ConfigDict(extra='allow', populate_by_name=True) +wait_for_admin_idle(timeout=60.0, poll_interval=1.0) ``` -### `type` {#cmem_client.components.graph_store.StoreInformation.type} +Poll the administrative status until no administrative operation is in progress. -```python -type: str -``` +Administrative operations such as import_bootstrap_data() or create_showcase_data() +are asynchronous on the server: the method call returns as soon as the operation has +been accepted, not once it has finished. Call this method afterward to wait until the +operation has actually completed before relying on its effects, for example before +reading the graphs it wrote. -The type/implementation of the graph store (e.g., "GRAPHDB", "TENTRIS"). +**Parameters:** -### `version` {#cmem_client.components.graph_store.StoreInformation.version} +- **timeout** (float) – Maximum seconds to wait before raising TimeoutError. +- **poll_interval** (float) – Seconds to wait between status checks. -```python -version: str -``` +**Raises:** -The version string of the graph store implementation. +- TimeoutError – If an administrative operation is still in progress after timeout seconds. +- HTTPError – If a status request fails. diff --git a/docs/develop/cmem-client-api/api-reference/components/explore/logs/index.md b/docs/develop/cmem-client-api/api-reference/components/explore/logs/index.md new file mode 100644 index 000000000..d242379a7 --- /dev/null +++ b/docs/develop/cmem-client-api/api-reference/components/explore/logs/index.md @@ -0,0 +1,120 @@ +--- +title: "cmem-client: components.explore.logs module" +description: "The in-memory log buffer of Explore (DataPlatform)." +tags: + - API + - Python + - cmem-client +--- + +# `cmem_client.components.explore.logs` {#cmem_client.components.explore.logs} + + + +The in-memory log buffer of Explore (DataPlatform). + +**Classes:** + +- [**ExploreLogs**](#cmem_client.components.explore.logs.ExploreLogs) – Read the in-memory log buffer of Explore: the most recent lines, and its state. + +## `ExploreLogs` {#cmem_client.components.explore.logs.ExploreLogs} + +Bases: _LogBuffer + +Read the in-memory log buffer of Explore: the most recent lines, and its state. + +Each Explore instance buffers its own lines, so a response describes exactly one +instance, and its sequence numbers are only meaningful together with the instance +identifier it reports. + +**Examples:** + +```pycon +>>> from cmem_client.client import Client +>>> client = Client.from_env() +>>> tail = client.explore.logs.get_tail(level="WARN", limit=10) +>>> [line.message for line in tail.lines] +``` + +**Functions:** + +- [**get_status**](#cmem_client.components.explore.logs.ExploreLogs.get_status) – Read the state of the in-memory log buffer of Explore. +- [**get_tail**](#cmem_client.components.explore.logs.ExploreLogs.get_tail) – Read the most recent log lines of Explore. + +**Attributes:** + +- [**logger**](#cmem_client.components.explore.logs.ExploreLogs.logger) – + +### `get_status` {#cmem_client.components.explore.logs.ExploreLogs.get_status} + +```python +get_status() +``` + +Read the state of the in-memory log buffer of Explore. + +Reports whether lines are captured at all, at which level, how many are held and +which loggers are excluded. This is the way to find out why a log read returns +fewer lines than expected, or none. + +**Returns:** + +- [LogBufferStatus](../../../models/logs/index.md#cmem_client.models.logs.LogBufferStatus) – The state and configuration of the buffer of the instance which answered. + +**Raises:** + +- HTTPStatusError – If the request fails, for example because the deployment +does not know the endpoint or the user is not authorized for it. +- LogsReadError – If the answer is not a log buffer status. + +### `get_tail` {#cmem_client.components.explore.logs.ExploreLogs.get_tail} + +```python +get_tail(since=None, limit=None, level=None, logger=None, contains=None, request_id=None) +``` + +Read the most recent log lines of Explore. + +The lines are returned oldest first, and every filter is applied by Explore +before the limit. Which end the limit cuts depends on `since`: without it Explore +answers the newest `limit` matching lines, and with it the oldest `limit` lines +after the cursor, so a follower pages forward instead of skipping ahead. + +To follow the log, pass the `last_sequence` of the previous response as `since` +of the next request. A `dropped` above zero in a response means the client fell +behind and those lines are no longer buffered. Sequence numbers belong to the +instance which answered, so a changed `instance_id` invalidates the cursor. + +**Parameters:** + +- **since** (int | None) – Return only lines with a sequence higher than this one. +- **limit** (int | None) – Maximum number of lines to return. Explore clamps this to its own +configured maximum rather than rejecting a larger value, so asking for an +arbitrarily high number is how to read as much as it will give. Without a +limit Explore answers its own default page, which is smaller than its +maximum. +- **level** ([LogLevel](../../../models/logs/index.md#cmem_client.models.logs.LogLevel) | str | None) – Minimum level a line has to be logged at. Asking for WARN returns +WARN and ERROR lines. +- **logger** (list[str] | None) – Return only lines of loggers starting with one of these prefixes. +- **contains** (list[str] | None) – Return only lines containing all of these substrings, ignoring +case. +- **request_id** (str | None) – Return only lines produced while handling the request with this +correlation identifier. Only lines logged on the request thread carry +one. + +**Returns:** + +- [LogTailResponse](../../../models/logs/index.md#cmem_client.models.logs.LogTailResponse) – The matching lines together with the state of the read. + +**Raises:** + +- HTTPStatusError – If the request fails, for example because a filter value is +not valid, the buffer is disabled or the user is not authorized. +- LogsReadError – If the answer is not a log buffer response. + +### `logger` {#cmem_client.components.explore.logs.ExploreLogs.logger} + +```python +logger = logging.getLogger(f'{self._client.logger.name}.{self.__class__.__name__}') +``` + diff --git a/docs/develop/cmem-client-api/api-reference/components/sparql_wrapper/index.md b/docs/develop/cmem-client-api/api-reference/components/explore/sparql/index.md similarity index 57% rename from docs/develop/cmem-client-api/api-reference/components/sparql_wrapper/index.md rename to docs/develop/cmem-client-api/api-reference/components/explore/sparql/index.md index 078714564..945b8cbe7 100644 --- a/docs/develop/cmem-client-api/api-reference/components/sparql_wrapper/index.md +++ b/docs/develop/cmem-client-api/api-reference/components/explore/sparql/index.md @@ -1,5 +1,5 @@ --- -title: "cmem-client: components.sparql_wrapper module" +title: "cmem-client: components.explore.sparql module" description: "SPARQL Wrapper for eccenca Corporate Memory" tags: - API @@ -7,7 +7,7 @@ tags: - cmem-client --- -# `cmem_client.components.sparql_wrapper` {#cmem_client.components.sparql_wrapper} +# `cmem_client.components.explore.sparql` {#cmem_client.components.explore.sparql} @@ -15,9 +15,9 @@ SPARQL Wrapper for eccenca Corporate Memory **Classes:** -- [**SPARQLWrapper**](#cmem_client.components.sparql_wrapper.SPARQLWrapper) – Sparql wrapper class +- [**SPARQLWrapper**](#cmem_client.components.explore.sparql.SPARQLWrapper) – Sparql wrapper class -## `SPARQLWrapper` {#cmem_client.components.sparql_wrapper.SPARQLWrapper} +## `SPARQLWrapper` {#cmem_client.components.explore.sparql.SPARQLWrapper} ```python SPARQLWrapper(sparql_endpoint, update_endpoint, client) @@ -29,20 +29,20 @@ Sparql wrapper class **Functions:** -- [**query**](#cmem_client.components.sparql_wrapper.SPARQLWrapper.query) – Query a SPARQL endpoint. -- [**update**](#cmem_client.components.sparql_wrapper.SPARQLWrapper.update) – Perform update SPARQL query. +- [**query**](#cmem_client.components.explore.sparql.SPARQLWrapper.query) – Query a SPARQL endpoint. +- [**update**](#cmem_client.components.explore.sparql.SPARQLWrapper.update) – Perform update SPARQL query. **Attributes:** -- [**logger**](#cmem_client.components.sparql_wrapper.SPARQLWrapper.logger) – +- [**logger**](#cmem_client.components.explore.sparql.SPARQLWrapper.logger) – -### `logger` {#cmem_client.components.sparql_wrapper.SPARQLWrapper.logger} +### `logger` {#cmem_client.components.explore.sparql.SPARQLWrapper.logger} ```python logger = logging.getLogger(f'{self._client.logger.name}.{self.__class__.__name__}') ``` -### `query` {#cmem_client.components.sparql_wrapper.SPARQLWrapper.query} +### `query` {#cmem_client.components.explore.sparql.SPARQLWrapper.query} ```python query(query, default_graph=None, named_graph=None, owl_imports_resolution=True) @@ -50,7 +50,7 @@ query(query, default_graph=None, named_graph=None, owl_imports_resolution=True) Query a SPARQL endpoint. -### `update` {#cmem_client.components.sparql_wrapper.SPARQLWrapper.update} +### `update` {#cmem_client.components.explore.sparql.SPARQLWrapper.update} ```python update(query, default_graph=None, named_graph=None) diff --git a/docs/develop/cmem-client-api/api-reference/components/explore/titles/index.md b/docs/develop/cmem-client-api/api-reference/components/explore/titles/index.md new file mode 100644 index 000000000..921755da4 --- /dev/null +++ b/docs/develop/cmem-client-api/api-reference/components/explore/titles/index.md @@ -0,0 +1,96 @@ +--- +title: "cmem-client: components.explore.titles module" +description: "Titles of resources, resolved by the Corporate Memory Explore title service." +tags: + - API + - Python + - cmem-client +--- + +# `cmem_client.components.explore.titles` {#cmem_client.components.explore.titles} + + + +Titles of resources, resolved by the Corporate Memory Explore title service. + +**Classes:** + +- [**ExploreTitles**](#cmem_client.components.explore.titles.ExploreTitles) – Resolve IRIs to human-readable titles, with one request per batch and a cache. + +## `ExploreTitles` {#cmem_client.components.explore.titles.ExploreTitles} + +```python +ExploreTitles(client) +``` + +Resolve IRIs to human-readable titles, with one request per batch and a cache. + +Explore resolves a title from the label of a resource, in the language it prefers, and +derives one from the IRI when there is no label. Every answer is cached per IRI and +context graph for as long as the client lives, so a title is requested only once. +There is no endpoint listing all titles, so this is a feature of the Explore component +rather than a repository. + +**Examples:** + +```pycon +>>> from cmem_client.client import Client +>>> client = Client.from_env() +>>> titles = client.explore.titles.get_many(["https://vocab.eccenca.com/shacl/", "urn:x:a"]) +>>> client.explore.titles.get("https://vocab.eccenca.com/shacl/").title +``` + +**Functions:** + +- [**get**](#cmem_client.components.explore.titles.ExploreTitles.get) – Get the title of one IRI. +- [**get_many**](#cmem_client.components.explore.titles.ExploreTitles.get_many) – Get the titles of several IRIs, with one request for all which are not cached yet. + +**Parameters:** + +- **client** ([Client](../../../client/index.md#cmem_client.client.Client)) – A configured Corporate Memory client instance. + +### `get` {#cmem_client.components.explore.titles.ExploreTitles.get} + +```python +get(iri, context_graph=None, refresh=False) +``` + +Get the title of one IRI. + +**Parameters:** + +- **iri** (str) – The IRI to resolve. +- **context_graph** (str | None) – The graph to read the label from, if not all graphs. +- **refresh** (bool) – Ask the service again instead of using a cached title. + +**Returns:** + +- [Title](../../../models/explore/index.md#cmem_client.models.explore.Title) – The title of the IRI. An IRI the service does not answer for gets itself as title. + +**Raises:** + +- HTTPStatusError – If the title service rejects the request. + +### `get_many` {#cmem_client.components.explore.titles.ExploreTitles.get_many} + +```python +get_many(iris, context_graph=None, refresh=False) +``` + +Get the titles of several IRIs, with one request for all which are not cached yet. + +**Parameters:** + +- **iris** (Iterable[str]) – The IRIs to resolve. Duplicates are resolved once. +- **context_graph** (str | None) – The graph to read the labels from, if not all graphs. +- **refresh** (bool) – Ask the service again for all of them instead of using cached titles. + +**Returns:** + +- dict[str, [Title](../../../models/explore/index.md#cmem_client.models.explore.Title)] – The title of every IRI, by IRI and in the order given. An IRI the service does +- dict[str, [Title](../../../models/explore/index.md#cmem_client.models.explore.Title)] – not answer for gets itself as title, and is asked for again next time. + +**Raises:** + +- HTTPStatusError – If the title service rejects the request. + diff --git a/docs/develop/cmem-client-api/api-reference/components/graphinsights/.pages b/docs/develop/cmem-client-api/api-reference/components/graphinsights/.pages new file mode 100644 index 000000000..9d556af07 --- /dev/null +++ b/docs/develop/cmem-client-api/api-reference/components/graphinsights/.pages @@ -0,0 +1,4 @@ +--- +title: graphinsights +nav: + - component: component diff --git a/docs/develop/cmem-client-api/api-reference/components/graphinsights/component/index.md b/docs/develop/cmem-client-api/api-reference/components/graphinsights/component/index.md new file mode 100644 index 000000000..fa10432cd --- /dev/null +++ b/docs/develop/cmem-client-api/api-reference/components/graphinsights/component/index.md @@ -0,0 +1,97 @@ +--- +title: "cmem-client: components.graphinsights.component module" +description: "Corporate Memory Graph Insights component." +tags: + - API + - Python + - cmem-client +--- + +# `cmem_client.components.graphinsights.component` {#cmem_client.components.graphinsights.component} + + + +Corporate Memory Graph Insights component. + +Graph Insights is an optional extension of Corporate Memory which computes statistics +snapshots of graphs. This component covers the extension itself; the snapshots are the +items of the [`GraphSnapshotsRepository`](../../../repositories/graph_snapshots/index.md#cmem_client.repositories.graph_snapshots.GraphSnapshotsRepository) +returned by `client.graph_snapshots`. + +The component is `client.graphinsights`. Until cmem-client 2.0.0, `client.graph_insights` +still returns the snapshot repository it used to be, with a `DeprecationWarning` naming +`client.graph_snapshots`. + +**Classes:** + +- [**GraphInsights**](#cmem_client.components.graphinsights.component.GraphInsights) – High-level interface for the Corporate Memory Graph Insights extension. + +## `GraphInsights` {#cmem_client.components.graphinsights.component.GraphInsights} + +```python +GraphInsights(client) +``` + +High-level interface for the Corporate Memory Graph Insights extension. + +Graph Insights is optional, so check `is_available()` before using its snapshots +through `client.graph_snapshots`. + +**Examples:** + +```pycon +>>> from cmem_client.client import Client +>>> client = Client.from_env() +>>> client.graphinsights.is_available() +``` + +**Functions:** + +- [**get_health**](#cmem_client.components.graphinsights.component.GraphInsights.get_health) – Report version and health of the Graph Insights extension. +- [**is_available**](#cmem_client.components.graphinsights.component.GraphInsights.is_available) – Tell whether the Graph Insights extension is active and the user may use it. + +**Attributes:** + +- [**logger**](#cmem_client.components.graphinsights.component.GraphInsights.logger) – + +**Parameters:** + +- **client** ([Client](../../../client/index.md#cmem_client.client.Client)) – A configured Corporate Memory client instance with +authentication and endpoint configuration. + +### `get_health` {#cmem_client.components.graphinsights.component.GraphInsights.get_health} + +```python +get_health() +``` + +Report version and health of the Graph Insights extension. + +An extension which is not available is reported as not applicable. Otherwise the +version and health come from its `/actuator/info` and `/actuator/health`; a build +of Graph Insights without these endpoints keeps version and health UNKNOWN and +names the failing request in the error, instead of being reported DOWN. + +**Returns:** + +- [GraphInsightsStatus](../../../models/status/index.md#cmem_client.models.status.GraphInsightsStatus) – The status of the extension, without raising. + +### `is_available` {#cmem_client.components.graphinsights.component.GraphInsights.is_available} + +```python +is_available() +``` + +Tell whether the Graph Insights extension is active and the user may use it. + +**Returns:** + +- bool – True if the extension is active and the user is allowed to use it. False +- bool – otherwise, including when the extension cannot be reached. + +### `logger` {#cmem_client.components.graphinsights.component.GraphInsights.logger} + +```python +logger = logging.getLogger(f'{self._client.logger.name}.{self.__class__.__name__}') +``` + diff --git a/docs/develop/cmem-client-api/api-reference/components/marketplace/.pages b/docs/develop/cmem-client-api/api-reference/components/marketplace/.pages new file mode 100644 index 000000000..c706425c2 --- /dev/null +++ b/docs/develop/cmem-client-api/api-reference/components/marketplace/.pages @@ -0,0 +1,4 @@ +--- +title: marketplace +nav: + - component: component diff --git a/docs/develop/cmem-client-api/api-reference/components/marketplace/component/index.md b/docs/develop/cmem-client-api/api-reference/components/marketplace/component/index.md new file mode 100644 index 000000000..86272bdcb --- /dev/null +++ b/docs/develop/cmem-client-api/api-reference/components/marketplace/component/index.md @@ -0,0 +1,412 @@ +--- +title: "cmem-client: components.marketplace.component module" +description: "eccenca Marketplace server integration." +tags: + - API + - Python + - cmem-client +--- + +# `cmem_client.components.marketplace.component` {#cmem_client.components.marketplace.component} + + + +eccenca Marketplace server integration. + +This module provides the Marketplace component for interacting with the eccenca +Marketplace server. The component handles package downloads, uploads and version queries, +abstracting the marketplace REST API into a convenient Python interface. + +The eccenca Marketplace is a central repository for distributing Corporate Memory +packages, including vocabularies, ontologies, and Python plugins. This component +enables automated package retrieval for installation and dependency resolution. +Packages and licensing each live in their own module of this package; the `Marketplace` +facade delegates to them. + +**Classes:** + +- [**Marketplace**](#cmem_client.components.marketplace.component.Marketplace) – Interface for eccenca Marketplace server operations. + +**Attributes:** + +- [**LICENSE_HEADER**](#cmem_client.components.marketplace.component.LICENSE_HEADER) – Header carrying the encrypted license on outbound marketplace requests. +- [**MARKETPLACE_CACHE_DIR**](#cmem_client.components.marketplace.component.MARKETPLACE_CACHE_DIR) – + +## `LICENSE_HEADER` {#cmem_client.components.marketplace.component.LICENSE_HEADER} + +```python +LICENSE_HEADER = _LICENSE_HEADER +``` + +Header carrying the encrypted license on outbound marketplace requests. + +## `MARKETPLACE_CACHE_DIR` {#cmem_client.components.marketplace.component.MARKETPLACE_CACHE_DIR} + +```python +MARKETPLACE_CACHE_DIR = xdg_cache_home() / 'eccenca-marketplace' +``` + +## `Marketplace` {#cmem_client.components.marketplace.component.Marketplace} + +```python +Marketplace(client, marketplace_url='https://eccenca.market', cache_dir=MARKETPLACE_CACHE_DIR, credentials=None, timeout=30, license_issuer_url=None) +``` + +Interface for eccenca Marketplace server operations. + +The Marketplace component provides methods for downloading, uploading and deleting +packages from the eccenca Marketplace server. It handles version resolution, package retrieval, +and writing downloaded content to the filesystem. + +**Attributes:** + +- **_client** ([Client](../../../client/index.md#cmem_client.client.Client)) – The Corporate Memory client instance used for HTTP communication. +- **_marketplace_url** ([HttpUrl](../../../models/url/index.md#cmem_client.models.url.HttpUrl)) – Default marketplace server URL for package operations. +- **_cache_dir** (Path | None) – Directory to use for cached downloads. + +**Functions:** + +- [**delete_package**](#cmem_client.components.marketplace.component.Marketplace.delete_package) – Delete a package from the marketplace server. +- [**download_package**](#cmem_client.components.marketplace.component.Marketplace.download_package) – Download a package from the marketplace server to a specified directory. +- [**get_available_packages**](#cmem_client.components.marketplace.component.Marketplace.get_available_packages) – Get the available packages from the marketplace server. +- [**get_available_packages_page**](#cmem_client.components.marketplace.component.Marketplace.get_available_packages_page) – Get a filtered, sorted and paginated page of packages from the marketplace server. +- [**get_dependencies_from_package**](#cmem_client.components.marketplace.component.Marketplace.get_dependencies_from_package) – Get the dependencies declared by a package version. +- [**get_health**](#cmem_client.components.marketplace.component.Marketplace.get_health) – Report version and health of the local marketplace, the one beside Corporate Memory. +- [**get_manifest_from_package**](#cmem_client.components.marketplace.component.Marketplace.get_manifest_from_package) – Get the manifest of a package version. +- [**get_marketplace_keycloak_token**](#cmem_client.components.marketplace.component.Marketplace.get_marketplace_keycloak_token) – Get the marketplace keycloak token from the marketplace server provided keycloak instance. +- [**get_package**](#cmem_client.components.marketplace.component.Marketplace.get_package) – Get the metadata of a single package from the marketplace server. +- [**get_versions_from_package**](#cmem_client.components.marketplace.component.Marketplace.get_versions_from_package) – Get the available versions of a package from the marketplace server. +- [**set_license_token**](#cmem_client.components.marketplace.component.Marketplace.set_license_token) – Set the license token (``x-eccenca-auth``) to send to a marketplace target. +- [**upload_package**](#cmem_client.components.marketplace.component.Marketplace.upload_package) – Upload a local package to the marketplace server. + +**Parameters:** + +- **client** ([Client](../../../client/index.md#cmem_client.client.Client)) – The Corporate Memory client instance. +- **marketplace_url** ([HttpUrl](../../../models/url/index.md#cmem_client.models.url.HttpUrl) | str) – Default marketplace server URL. Defaults to the public eccenca Marketplace. +- **cache_dir** (Path | None) – Directory to use for cached downloads. If set to None, caching is disabled. +- **credentials** ([BaseCredentials](../../../models/credentials/index.md#cmem_client.models.credentials.BaseCredentials) | None) – The credentials used to authenticate with the marketplace server. Setting this attribute +is needed for write operations on the server. +- **timeout** (int) – The timeout to wait for a response from the marketplace server. Defaults to 30 seconds. +- **license_issuer_url** ([HttpUrl](../../../models/url/index.md#cmem_client.models.url.HttpUrl) | str | None) – The local marketplace whose ``/api/session`` mints the encrypted license token +attached as ``x-eccenca-auth`` to outbound requests. Defaults to +``client.config.url_marketplace`` (the marketplace bundled beside Corporate Memory). + +### `cache_dir` {#cmem_client.components.marketplace.component.Marketplace.cache_dir} + +```python +cache_dir: Path | None +``` + +Get the cache directory. + +### `credentials` {#cmem_client.components.marketplace.component.Marketplace.credentials} + +```python +credentials: BaseCredentials | None +``` + +Get the marketplace credentials. + +### `delete_package` {#cmem_client.components.marketplace.component.Marketplace.delete_package} + +```python +delete_package(package_id) +``` + +Delete a package from the marketplace server. + +**Parameters:** + +- **package_id** (PackageIdentifier) – Marketplace package identifier of the package to be deleted. + +**Raises:** + +- MarketplaceAuthError – If the token could not be provided. +- MarketplaceDeleteError – If the deletion was rejected by the marketplace server. +- HTTPError – If the marketplace server request fails. + +### `download_package` {#cmem_client.components.marketplace.component.Marketplace.download_package} + +```python +download_package(package_id, path=None, package_version=None, use_cache=True) +``` + +Download a package from the marketplace server to a specified directory. + +Queries the marketplace server for available versions and downloads the +requested package version. If no version is specified, downloads the latest +available version. The package is saved with the naming convention: +{package_id}-v{version}.cpa + +If the package already exists in the cache, it will be reused instead of +re-downloading. + +**Parameters:** + +- **package_id** (PackageIdentifier) – Marketplace package identifier (e.g., "semanticarts-gist-vocab"). +- **path** (Path | None) – Target directory where the package will be saved. If None, uses the +cache directory. Must be a directory, not a file path. +- **package_version** (PackageVersionIdentifier | None) – Specific version to download. If None, downloads the latest version. +- **use_cache** (bool) – If True, use cached version if available instead of downloading. + +**Returns:** + +- Path – The full file path where the package was saved (e.g., +- Path – /path/to/cache/semanticarts-gist-vocab-v13.0.0.cpa). + +**Raises:** + +- MarketplaceReadError – If the marketplace server request fails or the package/version is not found. + +### `get_available_packages` {#cmem_client.components.marketplace.component.Marketplace.get_available_packages} + +```python +get_available_packages() +``` + +Get the available packages from the marketplace server. + +### `get_available_packages_page` {#cmem_client.components.marketplace.component.Marketplace.get_available_packages_page} + +```python +get_available_packages_page(package_type=None, search=None, tags=None, sort_by=SortField.package_id, sort_order=SortOrder.asc, offset=0, limit=20, package_license=None, language=None, dependency=None) +``` + +Get a filtered, sorted and paginated page of packages from the marketplace server. + +Marketplace servers older than the paginated route answer 404; for those the plain +package listing is fetched and filtered, sorted and paged here, so callers see the +same result shape regardless of the server version. A server that predates one of the +filters ignores it, so the result is then wider than asked for. + +**Parameters:** + +- **package_type** (str | None) – Restrict the result to one package type. +- **search** (str | None) – Case-insensitive substring matched against name, description and id. +- **tags** (list[str] | None) – Only packages carrying all of these tags. +- **sort_by** (SortField) – Metadata field to sort by. +- **sort_order** (SortOrder) – Sort direction. +- **offset** (int) – Number of items to skip. +- **limit** (int) – Maximum number of items to return. +- **package_license** (str | None) – Restrict the result to one license identifier. +- **language** (list[str] | None) – Only packages available in all of these languages. An empty list leaves +the language filter unset. +- **dependency** (str | None) – Restrict the result to packages depending on this package id. + +**Returns:** + +- PackageListResponse – The requested page, including paging position and facet counts. + +**Raises:** + +- MarketplaceReadError – If the marketplace server request fails. + +### `get_dependencies_from_package` {#cmem_client.components.marketplace.component.Marketplace.get_dependencies_from_package} + +```python +get_dependencies_from_package(package_id, package_version) +``` + +Get the dependencies declared by a package version. + +**Parameters:** + +- **package_id** (PackageIdentifier) – Marketplace package identifier. +- **package_version** (PackageVersionIdentifier) – The version to read the dependencies of. + +**Returns:** + +- list[ValidDependency] – The marketplace package and python package dependencies of that version. + +**Raises:** + +- MarketplaceReadError – If the package version is unknown or the request fails. + +### `get_health` {#cmem_client.components.marketplace.component.Marketplace.get_health} + +```python +get_health() +``` + +Report version and health of the local marketplace, the one beside Corporate Memory. + +Reads `api/info` of the marketplace at `client.config.url_marketplace`, which also +serves as its health probe; the configured `marketplace_url` does not matter here. + +**Returns:** + +- [MarketplaceStatus](../../../models/status/index.md#cmem_client.models.status.MarketplaceStatus) – The status of the local marketplace: UP with its version, or DOWN with the +- [MarketplaceStatus](../../../models/status/index.md#cmem_client.models.status.MarketplaceStatus) – error of a failing request, instead of raising. + +### `get_manifest_from_package` {#cmem_client.components.marketplace.component.Marketplace.get_manifest_from_package} + +```python +get_manifest_from_package(package_id, package_version) +``` + +Get the manifest of a package version. + +**Parameters:** + +- **package_id** (PackageIdentifier) – Marketplace package identifier. +- **package_version** (PackageVersionIdentifier) – The version to read the manifest of. + +**Returns:** + +- ValidManifest – The manifest of that version. + +**Raises:** + +- MarketplaceReadError – If the package version is unknown or the request fails. + +### `get_marketplace_keycloak_token` {#cmem_client.components.marketplace.component.Marketplace.get_marketplace_keycloak_token} + +```python +get_marketplace_keycloak_token(credentials=None) +``` + +Get the marketplace keycloak token from the marketplace server provided keycloak instance. + +This method first fetches the keycloak token URL needed from the marketplace server. +After this it fetches a token with the given credentials and provides it for +further authentication of protected routes. + +When no credentials parameter is given, it uses the class attribute of the marketplace component. + +**Parameters:** + +- **credentials** ([BaseCredentials](../../../models/credentials/index.md#cmem_client.models.credentials.BaseCredentials) | None) – Marketplace keycloak credentials. + +**Returns:** + +- str – The marketplace keycloak token. + +**Raises:** + +- MarketplaceAuthError – If no token could be provided. + +### `get_package` {#cmem_client.components.marketplace.component.Marketplace.get_package} + +```python +get_package(package_id) +``` + +Get the metadata of a single package from the marketplace server. + +**Parameters:** + +- **package_id** (PackageIdentifier) – Marketplace package identifier. + +**Returns:** + +- PackageMetadata – The package metadata of the latest version. + +**Raises:** + +- MarketplaceReadError – If the package is unknown or the request fails. + +### `get_versions_from_package` {#cmem_client.components.marketplace.component.Marketplace.get_versions_from_package} + +```python +get_versions_from_package(package_id) +``` + +Get the available versions of a package from the marketplace server. + +**Parameters:** + +- **package_id** (PackageIdentifier) – Marketplace package identifier. + +**Returns:** + +- list[PackageVersionIdentifier] – List of package versions available, newest first. + +**Raises:** + +- MarketplaceReadError – If the versions could not be retrieved from the +marketplace server. + +### `http` {#cmem_client.components.marketplace.component.Marketplace.http} + +```python +http: httpx.Client +``` + +Get the HTTP client instance for making API requests. + +Returns the configured HTTP client, creating it lazily on first access. +The client is pre-configured with the timeout value. Authentication headers +are added per-request by the individual methods that require them. + +**Returns:** + +- Client – The httpx.Client instance configured for the marketplace component. + +### `license_issuer_url` {#cmem_client.components.marketplace.component.Marketplace.license_issuer_url} + +```python +license_issuer_url: HttpUrl +``` + +Get the local marketplace URL used as the license-token issuer. + +### `logger` {#cmem_client.components.marketplace.component.Marketplace.logger} + +```python +logger = logging.getLogger(f'{self._client.logger.name}.{self.__class__.__name__}') +``` + +### `marketplace_url` {#cmem_client.components.marketplace.component.Marketplace.marketplace_url} + +```python +marketplace_url: HttpUrl +``` + +Get the marketplace server URL. + +### `set_license_token` {#cmem_client.components.marketplace.component.Marketplace.set_license_token} + +```python +set_license_token(token, marketplace_url=None) +``` + +Set the license token (``x-eccenca-auth``) to send to a marketplace target. + +Callers that can produce the encrypted license themselves -- a marketplace server holds +its own license and the target's public key -- use this instead of letting the component +fetch a token from the license issuer's ``/api/session``. + +**Parameters:** + +- **token** (str | None) – The encrypted license to attach, or None to record that there is none +(which keeps the header off without a lookup attempt). +- **marketplace_url** ([HttpUrl](../../../models/url/index.md#cmem_client.models.url.HttpUrl) | str | None) – The target this token is for. Defaults to the currently +configured marketplace URL. + +### `timeout` {#cmem_client.components.marketplace.component.Marketplace.timeout} + +```python +timeout: int +``` + +Get the marketplace timeout. + +### `upload_package` {#cmem_client.components.marketplace.component.Marketplace.upload_package} + +```python +upload_package(package_id, path) +``` + +Upload a local package to the marketplace server. + +**Parameters:** + +- **package_id** (PackageIdentifier) – Marketplace package identifier of the package to be uploaded. +- **path** (Path) – Path of the package to be uploaded. Must be a valid .cpa file. + +**Raises:** + +- MarketplaceAuthError – If the token could not be provided. +- MarketplaceWriteError – If the upload was rejected by the marketplace server. +- HTTPError – If the marketplace server request fails. + diff --git a/docs/develop/cmem-client-api/api-reference/components/marketplace/index.md b/docs/develop/cmem-client-api/api-reference/components/marketplace/index.md deleted file mode 100644 index fbcbc1564..000000000 --- a/docs/develop/cmem-client-api/api-reference/components/marketplace/index.md +++ /dev/null @@ -1,272 +0,0 @@ ---- -title: "cmem-client: components.marketplace module" -description: "eccenca Marketplace server integration." -tags: - - API - - Python - - cmem-client ---- - -# `cmem_client.components.marketplace` {#cmem_client.components.marketplace} - - - -eccenca Marketplace server integration. - -This module provides the Marketplace component for interacting with the eccenca -Marketplace server. The component handles package downloads, uploads and version queries, -abstracting the marketplace REST API into a convenient Python interface. - -The eccenca Marketplace is a central repository for distributing Corporate Memory -packages, including vocabularies, ontologies, and Python plugins. This component -enables automated package retrieval for installation and dependency resolution. - -**Classes:** - -- [**Marketplace**](#cmem_client.components.marketplace.Marketplace) – Interface for eccenca Marketplace server operations. - -**Attributes:** - -- [**LICENSE_HEADER**](#cmem_client.components.marketplace.LICENSE_HEADER) – Header carrying the encrypted license on outbound marketplace requests. -- [**MARKETPLACE_CACHE_DIR**](#cmem_client.components.marketplace.MARKETPLACE_CACHE_DIR) – - -## `LICENSE_HEADER` {#cmem_client.components.marketplace.LICENSE_HEADER} - -```python -LICENSE_HEADER = 'x-eccenca-auth' -``` - -Header carrying the encrypted license on outbound marketplace requests. - -## `MARKETPLACE_CACHE_DIR` {#cmem_client.components.marketplace.MARKETPLACE_CACHE_DIR} - -```python -MARKETPLACE_CACHE_DIR = xdg_cache_home() / 'eccenca-marketplace' -``` - -## `Marketplace` {#cmem_client.components.marketplace.Marketplace} - -```python -Marketplace(client, marketplace_url='https://eccenca.market', cache_dir=MARKETPLACE_CACHE_DIR, credentials=None, timeout=30, license_issuer_url=None) -``` - -Interface for eccenca Marketplace server operations. - -The Marketplace component provides methods for downloading, uploading and deleting -packages from the eccenca Marketplace server. It handles version resolution, package retrieval, -and writing downloaded content to the filesystem. - -**Attributes:** - -- **_client** ([Client](../../client/index.md#cmem_client.client.Client)) – The Corporate Memory client instance used for HTTP communication. -- **_marketplace_url** ([HttpUrl](../../models/url/index.md#cmem_client.models.url.HttpUrl)) – Default marketplace server URL for package operations. -- **_cache_dir** (Path | None) – Directory to use for cached downloads. - -**Functions:** - -- [**delete_package**](#cmem_client.components.marketplace.Marketplace.delete_package) – Delete a package from the marketplace server. -- [**download_package**](#cmem_client.components.marketplace.Marketplace.download_package) – Download a package from the marketplace server to a specified directory. -- [**get_available_packages**](#cmem_client.components.marketplace.Marketplace.get_available_packages) – Get the available packages from the marketplace server. -- [**get_marketplace_keycloak_token**](#cmem_client.components.marketplace.Marketplace.get_marketplace_keycloak_token) – Get the marketplace keycloak token from the marketplace server provided keycloak instance. -- [**get_versions_from_package**](#cmem_client.components.marketplace.Marketplace.get_versions_from_package) – Get the available versions of a package from the marketplace server. -- [**upload_package**](#cmem_client.components.marketplace.Marketplace.upload_package) – Upload a local package to the marketplace server. - -**Parameters:** - -- **client** ([Client](../../client/index.md#cmem_client.client.Client)) – The Corporate Memory client instance. -- **marketplace_url** ([HttpUrl](../../models/url/index.md#cmem_client.models.url.HttpUrl) | str) – Default marketplace server URL. Defaults to the public eccenca Marketplace. -- **cache_dir** (Path | None) – Directory to use for cached downloads. If set to None, caching is disabled. -- **credentials** ([BaseCredentials](../../models/credentials/index.md#cmem_client.models.credentials.BaseCredentials) | None) – The credentials used to authenticate with the marketplace server. Setting this attribute -is needed for write operations on the server. -- **timeout** (int) – The timeout to wait for a response from the marketplace server. Defaults to 30 seconds. -- **license_issuer_url** ([HttpUrl](../../models/url/index.md#cmem_client.models.url.HttpUrl) | str | None) – The local marketplace whose ``/api/session`` mints the encrypted license token -attached as ``x-eccenca-auth`` to outbound requests. Defaults to -``client.config.url_marketplace`` (the marketplace bundled beside Corporate Memory). - -### `cache_dir` {#cmem_client.components.marketplace.Marketplace.cache_dir} - -```python -cache_dir: Path | None -``` - -Get the cache directory. - -### `credentials` {#cmem_client.components.marketplace.Marketplace.credentials} - -```python -credentials: BaseCredentials | None -``` - -Get the marketplace credentials. - -### `delete_package` {#cmem_client.components.marketplace.Marketplace.delete_package} - -```python -delete_package(package_id) -``` - -Delete a package from the marketplace server. - -**Parameters:** - -- **package_id** (PackageIdentifier) – Marketplace package identifier of the package to be deleted. - -**Raises:** - -- [MarketplaceAuthError](../../exceptions/index.md#cmem_client.exceptions.MarketplaceAuthError) – If the token could not be provided. -- [MarketplaceDeleteError](../../exceptions/index.md#cmem_client.exceptions.MarketplaceDeleteError) – If the deletion was rejected by the marketplace server. -- HTTPError – If the marketplace server request fails. - -### `download_package` {#cmem_client.components.marketplace.Marketplace.download_package} - -```python -download_package(package_id, path=None, package_version=None, use_cache=True) -``` - -Download a package from the marketplace server to a specified directory. - -Queries the marketplace server for available versions and downloads the -requested package version. If no version is specified, downloads the latest -available version. The package is saved with the naming convention: -{package_id}-v{version}.cpa - -If the package already exists in the cache, it will be reused instead of -re-downloading. - -**Parameters:** - -- **package_id** (PackageIdentifier) – Marketplace package identifier (e.g., "semanticarts-gist-vocab"). -- **path** (Path | None) – Target directory where the package will be saved. If None, uses the -cache directory. Must be a directory, not a file path. -- **package_version** (PackageVersionIdentifier | None) – Specific version to download. If None, downloads the latest version. -- **use_cache** (bool) – If True, use cached version if available instead of downloading. - -**Returns:** - -- Path – The full file path where the package was saved (e.g., -- Path – /path/to/cache/semanticarts-gist-vocab-v13.0.0.cpa). - -**Raises:** - -- [MarketplaceReadError](../../exceptions/index.md#cmem_client.exceptions.MarketplaceReadError) – If the marketplace server request fails or the package/version is not found. - -### `get_available_packages` {#cmem_client.components.marketplace.Marketplace.get_available_packages} - -```python -get_available_packages() -``` - -Get the available packages from the marketplace server. - -### `get_marketplace_keycloak_token` {#cmem_client.components.marketplace.Marketplace.get_marketplace_keycloak_token} - -```python -get_marketplace_keycloak_token(credentials=None) -``` - -Get the marketplace keycloak token from the marketplace server provided keycloak instance. - -This method first fetches the keycloak token URL needed from the marketplace server. -After this it fetches a token with the given credentials and provides it for -further authentication of protected routes. - -When no credentials parameter is given, it uses the class attribute of the marketplace component. - -**Parameters:** - -- **credentials** ([BaseCredentials](../../models/credentials/index.md#cmem_client.models.credentials.BaseCredentials) | None) – Marketplace keycloak credentials. - -**Returns:** - -- str – The marketplace keycloak token. - -**Raises:** - -- [MarketplaceAuthError](../../exceptions/index.md#cmem_client.exceptions.MarketplaceAuthError) – If no token could be provided. - -### `get_versions_from_package` {#cmem_client.components.marketplace.Marketplace.get_versions_from_package} - -```python -get_versions_from_package(package_id) -``` - -Get the available versions of a package from the marketplace server. - -**Parameters:** - -- **package_id** (PackageIdentifier) – Marketplace package identifier. - -**Returns:** - -- list[PackageVersionIdentifier] – List of package versions available, newest first. - -**Raises:** - -- [MarketplaceReadError](../../exceptions/index.md#cmem_client.exceptions.MarketplaceReadError) – If the versions could not be retrieved from the -marketplace server. - -### `http` {#cmem_client.components.marketplace.Marketplace.http} - -```python -http: httpx.Client -``` - -Get the HTTP client instance for making API requests. - -Returns the configured HTTP client, creating it lazily on first access. -The client is pre-configured with the timeout value. Authentication headers -are added per-request by the individual methods that require them. - -**Returns:** - -- Client – The httpx.Client instance configured for the marketplace component. - -### `license_issuer_url` {#cmem_client.components.marketplace.Marketplace.license_issuer_url} - -```python -license_issuer_url: HttpUrl -``` - -Get the local marketplace URL used as the license-token issuer. - -### `logger` {#cmem_client.components.marketplace.Marketplace.logger} - -```python -logger = logging.getLogger(f'{self._client.logger.name}.{self.__class__.__name__}') -``` - -### `marketplace_url` {#cmem_client.components.marketplace.Marketplace.marketplace_url} - -```python -marketplace_url: HttpUrl -``` - -Get the marketplace server URL. - -### `timeout` {#cmem_client.components.marketplace.Marketplace.timeout} - -```python -timeout: int -``` - -Get the marketplace timeout. - -### `upload_package` {#cmem_client.components.marketplace.Marketplace.upload_package} - -```python -upload_package(package_id, path) -``` - -Upload a local package to the marketplace server. - -**Parameters:** - -- **package_id** (PackageIdentifier) – Marketplace package identifier of the package to be uploaded. -- **path** (Path) – Path of the package to be uploaded. Must be a valid .cpa file. - -**Raises:** - -- [MarketplaceAuthError](../../exceptions/index.md#cmem_client.exceptions.MarketplaceAuthError) – If the token could not be provided. -- [MarketplaceWriteError](../../exceptions/index.md#cmem_client.exceptions.MarketplaceWriteError) – If the upload was rejected by the marketplace server. -- HTTPError – If the marketplace server request fails. - diff --git a/docs/develop/cmem-client-api/api-reference/config/index.md b/docs/develop/cmem-client-api/api-reference/config/index.md index 46c4e9227..3ce1373d5 100644 --- a/docs/develop/cmem-client-api/api-reference/config/index.md +++ b/docs/develop/cmem-client-api/api-reference/config/index.md @@ -28,6 +28,7 @@ variables, making it suitable for different deployment environments. **Attributes:** - [**DEFAULT_CMEM_BASE_URI**](#cmem_client.config.DEFAULT_CMEM_BASE_URI) – +- [**logger**](#cmem_client.config.logger) – ## `Config` {#cmem_client.config.Config} @@ -155,7 +156,8 @@ DP_API_ENDPOINT (optional): DataPlatform API endpoint override. KEYCLOAK_BASE_URI (optional): Keycloak server URL override. KEYCLOAK_REALM_ID (optional): Keycloak realm identifier override. OAUTH_TOKEN_URI (optional): OAuth token endpoint override. -MARKETPLACE_BASE_URI (optional): Local marketplace (license issuer) override. +MARKETPLACE_API_ENDPOINT (optional): Local marketplace (license issuer) override. + Deprecated alias: MARKETPLACE_BASE_URI. SSL_VERIFY (optional): Set to ``"false"`` to disable SSL verification. REQUESTS_CA_BUNDLE (optional): Path to a custom CA bundle file. @@ -192,7 +194,8 @@ DP_API_ENDPOINT (optional): DataPlatform API endpoint override. KEYCLOAK_BASE_URI (optional): Keycloak server URL override. KEYCLOAK_REALM_ID (optional): Keycloak realm identifier override. OAUTH_TOKEN_URI (optional): OAuth token endpoint override. -MARKETPLACE_BASE_URI (optional): Local marketplace (license issuer) override. +MARKETPLACE_API_ENDPOINT (optional): Local marketplace (license issuer) override. + Deprecated alias: MARKETPLACE_BASE_URI. SSL_VERIFY (optional): SSL certificate verification flag. REQUESTS_CA_BUNDLE (optional): Path to a custom CA bundle file. @@ -257,6 +260,23 @@ it defaults to the base URL with '/dataplatform/' appended. - [HttpUrl](../models/url/index.md#cmem_client.models.url.HttpUrl) – The DataPlatform API endpoint URL. +### `url_graphinsights` {#cmem_client.config.Config.url_graphinsights} + +```python +url_graphinsights: HttpUrl +``` + +Get the Graph Insights extension URL. + +Returns the URL of the Graph Insights service deployed beside this Corporate +Memory instance, which serves its version and health on ``/actuator/info`` and +``/actuator/health``. If not explicitly set, it defaults to the base URL with +'/graphinsights/' appended. + +**Returns:** + +- [HttpUrl](../models/url/index.md#cmem_client.models.url.HttpUrl) – The Graph Insights extension URL. + ### `url_keycloak` {#cmem_client.config.Config.url_keycloak} ```python @@ -361,3 +381,9 @@ environments. Production deployments should always verify certificates. DEFAULT_CMEM_BASE_URI = 'http://docker.localhost' ``` +## `logger` {#cmem_client.config.logger} + +```python +logger = logging.getLogger(__name__) +``` + diff --git a/docs/develop/cmem-client-api/api-reference/exceptions/index.md b/docs/develop/cmem-client-api/api-reference/exceptions/index.md index ae72ba961..01e81cfc3 100644 --- a/docs/develop/cmem-client-api/api-reference/exceptions/index.md +++ b/docs/develop/cmem-client-api/api-reference/exceptions/index.md @@ -32,11 +32,14 @@ configuration, and repository operations. - [**GraphStoreBusyError**](#cmem_client.exceptions.GraphStoreBusyError) – Exception raised when a graph store admin operation is attempted while another is already in progress. - [**GraphVannMetadataConflictError**](#cmem_client.exceptions.GraphVannMetadataConflictError) – Exception raised when vann namespace metadata in the file conflicts with provided config. - [**GraphVannMetadataMissingError**](#cmem_client.exceptions.GraphVannMetadataMissingError) – Exception raised when vann namespace metadata is missing from an ontology file. +- [**LogsReadError**](#cmem_client.exceptions.LogsReadError) – Exception raised when a log buffer response cannot be interpreted. - [**MarketplaceAuthError**](#cmem_client.exceptions.MarketplaceAuthError) – Exception raised when a marketplace auth operation failed or is invalid. - [**MarketplaceDeleteError**](#cmem_client.exceptions.MarketplaceDeleteError) – Exception raised when a marketplace delete operation failed or is invalid. - [**MarketplacePackagesDeleteError**](#cmem_client.exceptions.MarketplacePackagesDeleteError) – Exception raised when a marketplace package deletion fails. - [**MarketplacePackagesExportError**](#cmem_client.exceptions.MarketplacePackagesExportError) – Exception raised when a marketplace packages export fails. +- [**MarketplacePackagesFileNotFoundError**](#cmem_client.exceptions.MarketplacePackagesFileNotFoundError) – Exception raised when an installed marketplace package declares no such file. - [**MarketplacePackagesImportError**](#cmem_client.exceptions.MarketplacePackagesImportError) – Exception raised when a marketplace package installation fails. +- [**MarketplacePackagesNotFoundError**](#cmem_client.exceptions.MarketplacePackagesNotFoundError) – Exception raised when a marketplace package is not installed. - [**MarketplaceReadError**](#cmem_client.exceptions.MarketplaceReadError) – Exception raised when a marketplace read operation failed or is invalid. - [**MarketplaceWriteError**](#cmem_client.exceptions.MarketplaceWriteError) – Exception raised when a marketplace write operation failed or is invalid. - [**ProjectExportError**](#cmem_client.exceptions.ProjectExportError) – Exception raised when a project export operation fails. @@ -132,6 +135,12 @@ Bases: [GraphImportError](#cmem_client.exceptions.GraphImportError) Exception raised when vann namespace metadata is missing from an ontology file. +## `LogsReadError` {#cmem_client.exceptions.LogsReadError} + +Bases: [BaseError](#cmem_client.exceptions.BaseError) + +Exception raised when a log buffer response cannot be interpreted. + ## `MarketplaceAuthError` {#cmem_client.exceptions.MarketplaceAuthError} Bases: [BaseError](#cmem_client.exceptions.BaseError) @@ -156,12 +165,24 @@ Bases: [BaseError](#cmem_client.exceptions.BaseError) Exception raised when a marketplace packages export fails. +## `MarketplacePackagesFileNotFoundError` {#cmem_client.exceptions.MarketplacePackagesFileNotFoundError} + +Bases: [RepositoryItemNotFoundError](#cmem_client.exceptions.RepositoryItemNotFoundError) + +Exception raised when an installed marketplace package declares no such file. + ## `MarketplacePackagesImportError` {#cmem_client.exceptions.MarketplacePackagesImportError} Bases: [RepositoryModificationError](#cmem_client.exceptions.RepositoryModificationError) Exception raised when a marketplace package installation fails. +## `MarketplacePackagesNotFoundError` {#cmem_client.exceptions.MarketplacePackagesNotFoundError} + +Bases: [RepositoryItemNotFoundError](#cmem_client.exceptions.RepositoryItemNotFoundError) + +Exception raised when a marketplace package is not installed. + ## `MarketplaceReadError` {#cmem_client.exceptions.MarketplaceReadError} Bases: [BaseError](#cmem_client.exceptions.BaseError) diff --git a/docs/develop/cmem-client-api/api-reference/index.md b/docs/develop/cmem-client-api/api-reference/index.md index d1b32c051..adf0169cc 100644 --- a/docs/develop/cmem-client-api/api-reference/index.md +++ b/docs/develop/cmem-client-api/api-reference/index.md @@ -27,11 +27,16 @@ tags: | [auth_provider.prefetched_token](auth_provider/prefetched_token/index.md) | Prefetched token authentication provider. | | [auth_provider.provided_token](auth_provider/provided_token/index.md) | Provided token authentication provider. | | [client](client/index.md) | Main API client for eccenca Corporate Memory. | +| [components.build.component](components/build/component/index.md) | Corporate Memory Build (DataIntegration) component. | +| [components.build.logs](components/build/logs/index.md) | The in-memory log buffer of Build (DataIntegration). | +| [components.build.workspace](components/build/workspace/index.md) | Corporate Memory DataIntegration (build) workspace management. | | [components.deployment](components/deployment/index.md) | Corporate Memory deployment status component. | -| [components.graph_store](components/graph_store/index.md) | Corporate Memory DataPlatform (explore) graph store management. | -| [components.marketplace](components/marketplace/index.md) | eccenca Marketplace server integration. | -| [components.sparql_wrapper](components/sparql_wrapper/index.md) | SPARQL Wrapper for eccenca Corporate Memory | -| [components.workspace](components/workspace/index.md) | Corporate Memory DataIntegration (build) workspace management. | +| [components.explore.component](components/explore/component/index.md) | Corporate Memory Explore (DataPlatform) component. | +| [components.explore.logs](components/explore/logs/index.md) | The in-memory log buffer of Explore (DataPlatform). | +| [components.explore.sparql](components/explore/sparql/index.md) | SPARQL Wrapper for eccenca Corporate Memory | +| [components.explore.titles](components/explore/titles/index.md) | Titles of resources, resolved by the Corporate Memory Explore title service. | +| [components.graphinsights.component](components/graphinsights/component/index.md) | Corporate Memory Graph Insights component. | +| [components.marketplace.component](components/marketplace/component/index.md) | eccenca Marketplace server integration. | | [config](config/index.md) | Configuration management for the Corporate Memory client. | | [exceptions](exceptions/index.md) | Custom exception classes for the cmem_client package. | | [logging_utils](logging_utils/index.md) | Logging utilities. | @@ -41,19 +46,23 @@ tags: | [models.credentials](models/credentials/index.md) | Models for the OAuth2 credentials. | | [models.dataset](models/dataset/index.md) | Corporate Memory dataset models for data integration. | | [models.error](models/error/index.md) | Error response models for Corporate Memory API error handling. | +| [models.explore](models/explore/index.md) | Models of the Corporate Memory Explore (DataPlatform) component. | | [models.graph](models/graph/index.md) | RDF graph models for Corporate Memory knowledge graphs. | | [models.graph_import](models/graph_import/index.md) | Graph import models for Corporate Memory. | | [models.graph_insight](models/graph_insight/index.md) | Graph Insight models for Corporate Memory. | | [models.item](models/item/index.md) | ImportItem base class and inherited classes | | [models.keycloak_client](models/keycloak_client/index.md) | Keycloak client models. | | [models.logging_config](models/logging_config/index.md) | Models for the configuration of the logging module | +| [models.logs](models/logs/index.md) | Corporate Memory log buffer models. | | [models.marshalling_plugins](models/marshalling_plugins/index.md) | Marshalling Plugin models | | [models.package](models/package/index.md) | Marketplace package models. | +| [models.paged_list](models/paged_list/index.md) | Models of paged list responses. | | [models.project](models/project/index.md) | Corporate Memory project models and metadata. | | [models.python_install](models/python_install/index.md) | Result models for Python package installation and plugin management operations. | | [models.python_package](models/python_package/index.md) | Python package models. | | [models.query_catalog](models/query_catalog/index.md) | Models for query catalog operations. | | [models.resource](models/resource/index.md) | A file resource model | +| [models.rule_block](models/rule_block/index.md) | Corporate Memory rule block models for data integration. | | [models.scheduler](models/scheduler/index.md) | Scheduler models | | [models.status](models/status/index.md) | Models for Corporate Memory aggregated status information. | | [models.task](models/task/index.md) | Task models for the DataIntegration task endpoint. | @@ -61,7 +70,7 @@ tags: | [models.url](models/url/index.md) | HTTP URL validation and manipulation utilities. | | [models.user](models/user/index.md) | Keycloak user and group models. | | [models.validation](models/validation/index.md) | Validation models for SHACL batch validation processes. | -| [models.variable](models/variable/index.md) | Corporate Memory project variable models for data integration. | +| [models.variable](models/variable/index.md) | Corporate Memory variable models for data integration. | | [models.vocabulary](models/vocabulary/index.md) | Vocabulary models for Corporate Memory vocabulary catalog. | | [models.workflow](models/workflow/index.md) | Workflow models | | [models.workspace_config](models/workspace_config/index.md) | Corporate Memory Explore workspace configuration models. | @@ -76,7 +85,7 @@ tags: | [repositories.datasets](repositories/datasets/index.md) | Repository for managing datasets in Corporate Memory. | | [repositories.files](repositories/files/index.md) | Repository for the file resources of DataIntegration projects. | | [repositories.graph_imports](repositories/graph_imports/index.md) | Repository for the `owl:imports` relations between named graphs. | -| [repositories.graph_insights](repositories/graph_insights/index.md) | Repository for the Graph Insights snapshots of Corporate Memory. | +| [repositories.graph_snapshots](repositories/graph_snapshots/index.md) | Repository for the Graph Insights snapshots of Corporate Memory. | | [repositories.graphs](repositories/graphs/index.md) | Repository for managing named graphs in Corporate Memory. | | [repositories.marketplace_packages](repositories/marketplace_packages/index.md) | Repository for the marketplace packages installed in Corporate Memory. | | [repositories.projects](repositories/projects/index.md) | Repository for managing DataIntegration projects. | @@ -87,6 +96,7 @@ tags: | [repositories.protocols.update_item](repositories/protocols/update_item/index.md) | Protocol interface for repository item update operations. | | [repositories.python_packages](repositories/python_packages/index.md) | Repository for the Python packages installed in DataIntegration. | | [repositories.queries](repositories/queries/index.md) | Repository for managing queries from the Corporate Memory query catalog. | +| [repositories.rule_blocks](repositories/rule_blocks/index.md) | Rule block repository. | | [repositories.schedulers](repositories/schedulers/index.md) | Repository for the workflow schedulers of DataIntegration. | | [repositories.user_accounts](repositories/user_accounts/index.md) | Repository for the Keycloak user accounts of a Corporate Memory deployment. | | [repositories.validations](repositories/validations/index.md) | Repository for the SHACL validation batches of Corporate Memory. | diff --git a/docs/develop/cmem-client-api/api-reference/logging_utils/index.md b/docs/develop/cmem-client-api/api-reference/logging_utils/index.md index 43f88aaf3..8adc39434 100644 --- a/docs/develop/cmem-client-api/api-reference/logging_utils/index.md +++ b/docs/develop/cmem-client-api/api-reference/logging_utils/index.md @@ -46,7 +46,7 @@ log_method(method, display_name=None) Wrapper to log entry and exit of methods using TRACE level. -Note: Don't use this on methods with sensitive information as they might get logged too +Note: Do not use this on methods with sensitive information as they might get logged too **Returns:** diff --git a/docs/develop/cmem-client-api/api-reference/models/.pages b/docs/develop/cmem-client-api/api-reference/models/.pages index 07f79b7d6..0987c03c3 100644 --- a/docs/develop/cmem-client-api/api-reference/models/.pages +++ b/docs/develop/cmem-client-api/api-reference/models/.pages @@ -7,19 +7,23 @@ nav: - credentials: credentials - dataset: dataset - error: error + - explore: explore - graph: graph - graph_import: graph_import - graph_insight: graph_insight - item: item - keycloak_client: keycloak_client - logging_config: logging_config + - logs: logs - marshalling_plugins: marshalling_plugins - package: package + - paged_list: paged_list - project: project - python_install: python_install - python_package: python_package - query_catalog: query_catalog - resource: resource + - rule_block: rule_block - scheduler: scheduler - status: status - task: task diff --git a/docs/develop/cmem-client-api/api-reference/models/access_condition/index.md b/docs/develop/cmem-client-api/api-reference/models/access_condition/index.md index ca0dd4305..d3f44c53a 100644 --- a/docs/develop/cmem-client-api/api-reference/models/access_condition/index.md +++ b/docs/develop/cmem-client-api/api-reference/models/access_condition/index.md @@ -223,7 +223,7 @@ An access condition result set **Attributes:** - [**content**](#cmem_client.models.access_condition.AccessConditionResultSet.content) (list[[AccessCondition](#cmem_client.models.access_condition.AccessCondition)]) – The access conditions on this page. -- [**page**](#cmem_client.models.access_condition.AccessConditionResultSet.page) ([PageDescription](../../repositories/base/paged_list/index.md#cmem_client.repositories.base.paged_list.PageDescription)) – Which page this is and how many there are in total. +- [**page**](#cmem_client.models.access_condition.AccessConditionResultSet.page) ([PageDescription](../../models/paged_list/index.md#cmem_client.models.paged_list.PageDescription)) – Which page this is and how many there are in total. ### `content` {#cmem_client.models.access_condition.AccessConditionResultSet.content} diff --git a/docs/develop/cmem-client-api/api-reference/models/explore/index.md b/docs/develop/cmem-client-api/api-reference/models/explore/index.md new file mode 100644 index 000000000..9439c8dbe --- /dev/null +++ b/docs/develop/cmem-client-api/api-reference/models/explore/index.md @@ -0,0 +1,172 @@ +--- +title: "cmem-client: models.explore module" +description: "Models of the Corporate Memory Explore (DataPlatform) component." +tags: + - API + - Python + - cmem-client +--- + +# `cmem_client.models.explore` {#cmem_client.models.explore} + + + +Models of the Corporate Memory Explore (DataPlatform) component. + +**Classes:** + +- [**AdminOperationStatus**](#cmem_client.models.explore.AdminOperationStatus) – Current administrative operation reported by GET /api/admin/status. +- [**StoreInformation**](#cmem_client.models.explore.StoreInformation) – Information about the graph store instance and its capabilities. +- [**Title**](#cmem_client.models.explore.Title) – The human-readable title of an IRI, as the Explore title service resolves it. + +## `AdminOperationStatus` {#cmem_client.models.explore.AdminOperationStatus} + +Bases: StrEnum + +Current administrative operation reported by GET /api/admin/status. + +**Attributes:** + +- [**BACKUP**](#cmem_client.models.explore.AdminOperationStatus.BACKUP) – +- [**BOOTSTRAP**](#cmem_client.models.explore.AdminOperationStatus.BOOTSTRAP) – +- [**DROP_ALL_GRAPHS**](#cmem_client.models.explore.AdminOperationStatus.DROP_ALL_GRAPHS) – +- [**NONE**](#cmem_client.models.explore.AdminOperationStatus.NONE) – +- [**RESTORE**](#cmem_client.models.explore.AdminOperationStatus.RESTORE) – +- [**SHOWCASE**](#cmem_client.models.explore.AdminOperationStatus.SHOWCASE) – + +### `BACKUP` {#cmem_client.models.explore.AdminOperationStatus.BACKUP} + +```python +BACKUP = 'BACKUP' +``` + +### `BOOTSTRAP` {#cmem_client.models.explore.AdminOperationStatus.BOOTSTRAP} + +```python +BOOTSTRAP = 'BOOTSTRAP' +``` + +### `DROP_ALL_GRAPHS` {#cmem_client.models.explore.AdminOperationStatus.DROP_ALL_GRAPHS} + +```python +DROP_ALL_GRAPHS = 'DROP_ALL_GRAPHS' +``` + +### `NONE` {#cmem_client.models.explore.AdminOperationStatus.NONE} + +```python +NONE = 'NONE' +``` + +### `RESTORE` {#cmem_client.models.explore.AdminOperationStatus.RESTORE} + +```python +RESTORE = 'RESTORE' +``` + +### `SHOWCASE` {#cmem_client.models.explore.AdminOperationStatus.SHOWCASE} + +```python +SHOWCASE = 'SHOWCASE' +``` + +## `StoreInformation` {#cmem_client.models.explore.StoreInformation} + +Bases: [Model](../../models/base/index.md#cmem_client.models.base.Model) + +Information about the graph store instance and its capabilities. + +This model represents metadata about the DataPlatform graph store, +including the store type and version information. This information +is useful for compatibility checks, monitoring, and debugging. + +**Attributes:** + +- [**type**](#cmem_client.models.explore.StoreInformation.type) (str) – The type of graph store (e.g., "GRAPHDB", "TENTRIS"). +- [**version**](#cmem_client.models.explore.StoreInformation.version) (str) – The version string of the graph store implementation. + +### `model_config` {#cmem_client.models.explore.StoreInformation.model_config} + +```python +model_config = ConfigDict(extra='allow', populate_by_name=True) +``` + +### `type` {#cmem_client.models.explore.StoreInformation.type} + +```python +type: str +``` + +The type/implementation of the graph store (e.g., "GRAPHDB", "TENTRIS"). + +### `version` {#cmem_client.models.explore.StoreInformation.version} + +```python +version: str +``` + +The version string of the graph store implementation. + +## `Title` {#cmem_client.models.explore.Title} + +Bases: [Model](../../models/base/index.md#cmem_client.models.base.Model) + +The human-readable title of an IRI, as the Explore title service resolves it. + +A resource with a label gets that label; any other IRI gets a title derived from the +IRI itself, marked with `from_iri`. + +**Attributes:** + +- [**iri**](#cmem_client.models.explore.Title.iri) (str) – The IRI the title belongs to. +- [**title**](#cmem_client.models.explore.Title.title) (str) – The label of the resource, or a title derived from the IRI. +- [**lang**](#cmem_client.models.explore.Title.lang) (str | None) – The language tag of the label, if it has one. +- [**from_iri**](#cmem_client.models.explore.Title.from_iri) (bool) – Whether the title was derived from the IRI instead of a label. +- [**predicate**](#cmem_client.models.explore.Title.predicate) (str | None) – The property the label was read from, if there is a label. + +### `from_iri` {#cmem_client.models.explore.Title.from_iri} + +```python +from_iri: bool = Field(default=False, alias='fromIri') +``` + +Whether the title was derived from the IRI instead of a label. + +### `iri` {#cmem_client.models.explore.Title.iri} + +```python +iri: str +``` + +The IRI the title belongs to. + +### `lang` {#cmem_client.models.explore.Title.lang} + +```python +lang: str | None = None +``` + +The language tag of the label, if it has one. + +### `model_config` {#cmem_client.models.explore.Title.model_config} + +```python +model_config = ConfigDict(extra='allow', populate_by_name=True) +``` + +### `predicate` {#cmem_client.models.explore.Title.predicate} + +```python +predicate: str | None = None +``` + +The property the label was read from, if there is a label. + +### `title` {#cmem_client.models.explore.Title.title} + +```python +title: str +``` + +The label of the resource, or a title derived from the IRI. + diff --git a/docs/develop/cmem-client-api/api-reference/models/graph/index.md b/docs/develop/cmem-client-api/api-reference/models/graph/index.md index d79388711..c05fec6f3 100644 --- a/docs/develop/cmem-client-api/api-reference/models/graph/index.md +++ b/docs/develop/cmem-client-api/api-reference/models/graph/index.md @@ -19,11 +19,13 @@ the primary storage units for knowledge graphs. The Graph model includes metadata about graph permissions, assigned semantic classes, and access control, providing the foundation for graph-based -operations in the explore APIs. +operations in the explore APIs. GraphFileSerialization describes an RDF serialization +the graphs repository imports and exports. **Classes:** - [**Graph**](#cmem_client.models.graph.Graph) – A graph +- [**GraphFileSerialization**](#cmem_client.models.graph.GraphFileSerialization) – Supported graph format description - [**GraphLabel**](#cmem_client.models.graph.GraphLabel) – Label metadata for a graph returned by the /graphs/list endpoint. ## `Graph` {#cmem_client.models.graph.Graph} @@ -39,6 +41,9 @@ A graph decided by the access conditions of the deployment. - [**assigned_classes**](#cmem_client.models.graph.Graph.assigned_classes) (list[str]) – IRIs of the classes assigned to the graph, which is how Corporate Memory tells a vocabulary from a data graph or a shape graph. +- [**di_project_graph**](#cmem_client.models.graph.Graph.di_project_graph) (bool) – Whether the graph is a DataIntegration project graph. +- [**system_resource**](#cmem_client.models.graph.Graph.system_resource) (bool) – Whether the graph is a system resource, e.g. a shape or +query catalog graph. - [**label**](#cmem_client.models.graph.Graph.label) ([GraphLabel](#cmem_client.models.graph.GraphLabel) | None) – Label of the graph, or ``None`` if it carries none. **Functions:** @@ -51,6 +56,12 @@ Corporate Memory tells a vocabulary from a data graph or a shape graph. assigned_classes: list[str] = Field(alias='assignedClasses') ``` +### `di_project_graph` {#cmem_client.models.graph.Graph.di_project_graph} + +```python +di_project_graph: bool = Field(alias='diProjectGraph') +``` + ### `get_id` {#cmem_client.models.graph.Graph.get_id} ```python @@ -77,12 +88,79 @@ label: GraphLabel | None = None model_config = ConfigDict(extra='allow', populate_by_name=True) ``` +### `system_resource` {#cmem_client.models.graph.Graph.system_resource} + +```python +system_resource: bool = Field(alias='systemResource') +``` + ### `writeable` {#cmem_client.models.graph.Graph.writeable} ```python writeable: bool ``` +## `GraphFileSerialization` {#cmem_client.models.graph.GraphFileSerialization} + +Bases: [Model](../../models/base/index.md#cmem_client.models.base.Model) + +Supported graph format description + +**Attributes:** + +- [**mime_type**](#cmem_client.models.graph.GraphFileSerialization.mime_type) (str) – MIME type of the serialization, sent as ``Content-Type`` on import and as +``Accept`` on export. +- [**file_extensions**](#cmem_client.models.graph.GraphFileSerialization.file_extensions) (list[str]) – File extensions mapped to this serialization, used by +``guess_file_type()`` to detect the format of a path. +- [**encoding**](#cmem_client.models.graph.GraphFileSerialization.encoding) (str | None) – Content encoding of the file, sent as ``Content-Encoding`` on import when set. +- [**known_not_supporters**](#cmem_client.models.graph.GraphFileSerialization.known_not_supporters) (list[str]) – Store types known not to support this serialization, matched against +the type reported by the graph store. The client does not enforce this; the test suite +uses it to skip combinations a store cannot handle. +- [**export_supported**](#cmem_client.models.graph.GraphFileSerialization.export_supported) (bool) – Whether graphs can be exported in this serialization. +- [**import_supported**](#cmem_client.models.graph.GraphFileSerialization.import_supported) (bool) – Whether graphs can be imported from this serialization. + +### `encoding` {#cmem_client.models.graph.GraphFileSerialization.encoding} + +```python +encoding: str | None = None +``` + +### `export_supported` {#cmem_client.models.graph.GraphFileSerialization.export_supported} + +```python +export_supported: bool = True +``` + +### `file_extensions` {#cmem_client.models.graph.GraphFileSerialization.file_extensions} + +```python +file_extensions: list[str] +``` + +### `import_supported` {#cmem_client.models.graph.GraphFileSerialization.import_supported} + +```python +import_supported: bool = True +``` + +### `known_not_supporters` {#cmem_client.models.graph.GraphFileSerialization.known_not_supporters} + +```python +known_not_supporters: list[str] = Field(default_factory=list) +``` + +### `mime_type` {#cmem_client.models.graph.GraphFileSerialization.mime_type} + +```python +mime_type: str +``` + +### `model_config` {#cmem_client.models.graph.GraphFileSerialization.model_config} + +```python +model_config = ConfigDict(extra='allow', populate_by_name=True) +``` + ## `GraphLabel` {#cmem_client.models.graph.GraphLabel} Bases: [Model](../../models/base/index.md#cmem_client.models.base.Model) diff --git a/docs/develop/cmem-client-api/api-reference/models/graph_insight/index.md b/docs/develop/cmem-client-api/api-reference/models/graph_insight/index.md index 5bcbc6c17..8b700dc42 100644 --- a/docs/develop/cmem-client-api/api-reference/models/graph_insight/index.md +++ b/docs/develop/cmem-client-api/api-reference/models/graph_insight/index.md @@ -14,7 +14,7 @@ tags: Graph Insight models for Corporate Memory. Graph Insight is the semspect extension, which keeps its own indexed snapshot of the -graphs it explores. The snapshots are the items of ``client.graph_insights``, keyed by +graphs it explores. The snapshots are the items of ``client.graph_snapshots``, keyed by their database ID. That repository does not fetch on creation, so call ``fetch_data()`` before iterating it. diff --git a/docs/develop/cmem-client-api/api-reference/models/logs/index.md b/docs/develop/cmem-client-api/api-reference/models/logs/index.md new file mode 100644 index 000000000..9f8982209 --- /dev/null +++ b/docs/develop/cmem-client-api/api-reference/models/logs/index.md @@ -0,0 +1,319 @@ +--- +title: "cmem-client: models.logs module" +description: "Corporate Memory log buffer models." +tags: + - API + - Python + - cmem-client +--- + +# `cmem_client.models.logs` {#cmem_client.models.logs} + + + +Corporate Memory log buffer models. + +Models for the in-memory log buffer API of a Corporate Memory service +(`GET /api/admin/logs` and `GET /api/admin/logs/status`). Each instance of a +service keeps its own buffer and answers only with its own lines, which is why +the responses carry an instance identifier and a per instance sequence number. + +**Classes:** + +- [**LogBufferStatus**](#cmem_client.models.logs.LogBufferStatus) – State and configuration of the in-memory log buffer of one instance. +- [**LogLevel**](#cmem_client.models.logs.LogLevel) – Log level a line can be requested at, ordered from least to most severe. +- [**LogLine**](#cmem_client.models.logs.LogLine) – A single buffered log line. +- [**LogTailResponse**](#cmem_client.models.logs.LogTailResponse) – The most recent buffered log lines of one instance, with the state of that read. + +## `LogBufferStatus` {#cmem_client.models.logs.LogBufferStatus} + +Bases: [Model](../../models/base/index.md#cmem_client.models.base.Model) + +State and configuration of the in-memory log buffer of one instance. + +**Attributes:** + +- [**enabled**](#cmem_client.models.logs.LogBufferStatus.enabled) (bool) – Whether the buffer is enabled in this deployment. +- [**attached**](#cmem_client.models.logs.LogBufferStatus.attached) (bool) – Whether the buffer is attached to the logging framework and therefore +actually capturing. +- [**capture_level**](#cmem_client.models.logs.LogBufferStatus.capture_level) (str | None) – The minimum level which is captured, regardless of the configured +logging levels. +- [**capacity**](#cmem_client.models.logs.LogBufferStatus.capacity) (int) – How many lines the buffer holds before the oldest ones are discarded. +- [**max_message_chars**](#cmem_client.models.logs.LogBufferStatus.max_message_chars) (int) – Length a message is truncated to before it is buffered. +- [**size**](#cmem_client.models.logs.LogBufferStatus.size) (int) – How many lines are buffered at the moment. +- [**first_sequence**](#cmem_client.models.logs.LogBufferStatus.first_sequence) (int) – Oldest sequence still buffered. +- [**last_sequence**](#cmem_client.models.logs.LogBufferStatus.last_sequence) (int) – Highest sequence buffered. +- [**instance_id**](#cmem_client.models.logs.LogBufferStatus.instance_id) (str) – The instance this status describes. +- [**excluded_loggers**](#cmem_client.models.logs.LogBufferStatus.excluded_loggers) (list[str]) – Loggers which are never captured, because they carry user data +or credentials. +- [**non_additive_loggers**](#cmem_client.models.logs.LogBufferStatus.non_additive_loggers) (list[str]) – Loggers which do not propagate to the buffer through their +parent. +- [**request_id_mdc_key**](#cmem_client.models.logs.LogBufferStatus.request_id_mdc_key) (str | None) – The mapped diagnostic context key the correlation identifier +is taken from. + +### `attached` {#cmem_client.models.logs.LogBufferStatus.attached} + +```python +attached: bool +``` + +### `capacity` {#cmem_client.models.logs.LogBufferStatus.capacity} + +```python +capacity: int +``` + +### `capture_level` {#cmem_client.models.logs.LogBufferStatus.capture_level} + +```python +capture_level: str | None = Field(alias='captureLevel', default=None) +``` + +### `enabled` {#cmem_client.models.logs.LogBufferStatus.enabled} + +```python +enabled: bool +``` + +### `excluded_loggers` {#cmem_client.models.logs.LogBufferStatus.excluded_loggers} + +```python +excluded_loggers: list[str] = Field(alias='excludedLoggers', default_factory=list) +``` + +### `first_sequence` {#cmem_client.models.logs.LogBufferStatus.first_sequence} + +```python +first_sequence: int = Field(alias='firstSequence') +``` + +### `instance_id` {#cmem_client.models.logs.LogBufferStatus.instance_id} + +```python +instance_id: str = Field(alias='instanceId') +``` + +### `last_sequence` {#cmem_client.models.logs.LogBufferStatus.last_sequence} + +```python +last_sequence: int = Field(alias='lastSequence') +``` + +### `max_message_chars` {#cmem_client.models.logs.LogBufferStatus.max_message_chars} + +```python +max_message_chars: int = Field(alias='maxMessageChars') +``` + +### `model_config` {#cmem_client.models.logs.LogBufferStatus.model_config} + +```python +model_config = ConfigDict(extra='allow', populate_by_name=True) +``` + +### `non_additive_loggers` {#cmem_client.models.logs.LogBufferStatus.non_additive_loggers} + +```python +non_additive_loggers: list[str] = Field(alias='nonAdditiveLoggers', default_factory=list) +``` + +### `request_id_mdc_key` {#cmem_client.models.logs.LogBufferStatus.request_id_mdc_key} + +```python +request_id_mdc_key: str | None = Field(alias='requestIdMdcKey', default=None) +``` + +### `size` {#cmem_client.models.logs.LogBufferStatus.size} + +```python +size: int +``` + +## `LogLevel` {#cmem_client.models.logs.LogLevel} + +Bases: StrEnum + +Log level a line can be requested at, ordered from least to most severe. + +The level of a request is a minimum: asking for WARN returns the lines logged +at WARN as well as the ones logged at ERROR. + +**Attributes:** + +- [**DEBUG**](#cmem_client.models.logs.LogLevel.DEBUG) – +- [**ERROR**](#cmem_client.models.logs.LogLevel.ERROR) – +- [**INFO**](#cmem_client.models.logs.LogLevel.INFO) – +- [**TRACE**](#cmem_client.models.logs.LogLevel.TRACE) – +- [**WARN**](#cmem_client.models.logs.LogLevel.WARN) – + +### `DEBUG` {#cmem_client.models.logs.LogLevel.DEBUG} + +```python +DEBUG = 'DEBUG' +``` + +### `ERROR` {#cmem_client.models.logs.LogLevel.ERROR} + +```python +ERROR = 'ERROR' +``` + +### `INFO` {#cmem_client.models.logs.LogLevel.INFO} + +```python +INFO = 'INFO' +``` + +### `TRACE` {#cmem_client.models.logs.LogLevel.TRACE} + +```python +TRACE = 'TRACE' +``` + +### `WARN` {#cmem_client.models.logs.LogLevel.WARN} + +```python +WARN = 'WARN' +``` + +## `LogLine` {#cmem_client.models.logs.LogLine} + +Bases: [Model](../../models/base/index.md#cmem_client.models.base.Model) + +A single buffered log line. + +**Attributes:** + +- [**sequence**](#cmem_client.models.logs.LogLine.sequence) (int) – Position of this line in the log stream of the answering instance. +- [**timestamp**](#cmem_client.models.logs.LogLine.timestamp) (datetime) – When the line was logged. +- [**level**](#cmem_client.models.logs.LogLine.level) (str) – Level the line was logged at. Not restricted to LogLevel, because a +service may log at a level which cannot be requested. +- [**logger**](#cmem_client.models.logs.LogLine.logger) (str) – Name of the logger which produced the line. +- [**thread**](#cmem_client.models.logs.LogLine.thread) (str | None) – Thread the line was logged on, if the service reported one. +- [**message**](#cmem_client.models.logs.LogLine.message) (str) – The log message. +- [**throwable**](#cmem_client.models.logs.LogLine.throwable) (str | None) – The rendered exception, if the line carried one. +- [**request_id**](#cmem_client.models.logs.LogLine.request_id) (str | None) – Correlation identifier of the request which produced this line. The +same value is returned in the X-Request-Id header and, for a failed request, +in its problem detail. Lines logged off the request thread carry none. + +### `level` {#cmem_client.models.logs.LogLine.level} + +```python +level: str +``` + +### `logger` {#cmem_client.models.logs.LogLine.logger} + +```python +logger: str +``` + +### `message` {#cmem_client.models.logs.LogLine.message} + +```python +message: str +``` + +### `model_config` {#cmem_client.models.logs.LogLine.model_config} + +```python +model_config = ConfigDict(extra='allow', populate_by_name=True) +``` + +### `request_id` {#cmem_client.models.logs.LogLine.request_id} + +```python +request_id: str | None = Field(alias='requestId', default=None) +``` + +### `sequence` {#cmem_client.models.logs.LogLine.sequence} + +```python +sequence: int +``` + +### `thread` {#cmem_client.models.logs.LogLine.thread} + +```python +thread: str | None = None +``` + +### `throwable` {#cmem_client.models.logs.LogLine.throwable} + +```python +throwable: str | None = None +``` + +### `timestamp` {#cmem_client.models.logs.LogLine.timestamp} + +```python +timestamp: datetime +``` + +## `LogTailResponse` {#cmem_client.models.logs.LogTailResponse} + +Bases: [Model](../../models/base/index.md#cmem_client.models.base.Model) + +The most recent buffered log lines of one instance, with the state of that read. + +**Attributes:** + +- [**server_time**](#cmem_client.models.logs.LogTailResponse.server_time) (int) – Epoch milliseconds the response was assembled at. +- [**instance_id**](#cmem_client.models.logs.LogTailResponse.instance_id) (str) – The instance which answered. Sequence numbers are only meaningful +together with this value. +- [**first_sequence**](#cmem_client.models.logs.LogTailResponse.first_sequence) (int) – Oldest sequence still buffered. +- [**last_sequence**](#cmem_client.models.logs.LogTailResponse.last_sequence) (int) – Highest sequence examined. Pass it as the `since` of the next +request to receive only what has been logged in between. +- [**truncated**](#cmem_client.models.logs.LogTailResponse.truncated) (bool) – Whether the limit cut the result short. +- [**dropped**](#cmem_client.models.logs.LogTailResponse.dropped) (int) – How many lines were lost between the requested `since` and what the +buffer still holds. +- [**lines**](#cmem_client.models.logs.LogTailResponse.lines) (list[[LogLine](#cmem_client.models.logs.LogLine)]) – The matching lines, oldest first. + +### `dropped` {#cmem_client.models.logs.LogTailResponse.dropped} + +```python +dropped: int = 0 +``` + +### `first_sequence` {#cmem_client.models.logs.LogTailResponse.first_sequence} + +```python +first_sequence: int = Field(alias='firstSequence') +``` + +### `instance_id` {#cmem_client.models.logs.LogTailResponse.instance_id} + +```python +instance_id: str = Field(alias='instanceId') +``` + +### `last_sequence` {#cmem_client.models.logs.LogTailResponse.last_sequence} + +```python +last_sequence: int = Field(alias='lastSequence') +``` + +### `lines` {#cmem_client.models.logs.LogTailResponse.lines} + +```python +lines: list[LogLine] = Field(default_factory=list) +``` + +### `model_config` {#cmem_client.models.logs.LogTailResponse.model_config} + +```python +model_config = ConfigDict(extra='allow', populate_by_name=True) +``` + +### `server_time` {#cmem_client.models.logs.LogTailResponse.server_time} + +```python +server_time: int = Field(alias='serverTime') +``` + +### `truncated` {#cmem_client.models.logs.LogTailResponse.truncated} + +```python +truncated: bool = False +``` + diff --git a/docs/develop/cmem-client-api/api-reference/models/marshalling_plugins/index.md b/docs/develop/cmem-client-api/api-reference/models/marshalling_plugins/index.md index de2d9a039..ffd78729f 100644 --- a/docs/develop/cmem-client-api/api-reference/models/marshalling_plugins/index.md +++ b/docs/develop/cmem-client-api/api-reference/models/marshalling_plugins/index.md @@ -15,8 +15,8 @@ Marshalling Plugin models A marshalling plugin defines the file format a DataIntegration workspace is written to and read from. The plugins a deployment offers are returned by -``client.workspace.get_marshalling_plugins()``, and their ``id`` is what -``client.workspace.export_to_zip()`` and ``import_from_zip()`` expect. +``client.build.workspace.get_marshalling_plugins()``, and their ``id`` is what +``client.build.workspace.export_to_zip()`` and ``import_from_zip()`` expect. **Classes:** diff --git a/docs/develop/cmem-client-api/api-reference/models/package/index.md b/docs/develop/cmem-client-api/api-reference/models/package/index.md index 682b5907e..5869bde4b 100644 --- a/docs/develop/cmem-client-api/api-reference/models/package/index.md +++ b/docs/develop/cmem-client-api/api-reference/models/package/index.md @@ -46,6 +46,14 @@ a package installed before this was recorded. - [**get_id**](#cmem_client.models.package.Package.get_id) – Get the package identifier. +### `dependencies` {#cmem_client.models.package.Package.dependencies} + +```python +dependencies: list[ValidDependency] +``` + +The dependencies declared by the installed version. + ### `get_id` {#cmem_client.models.package.Package.get_id} ```python @@ -64,6 +72,14 @@ Get the package identifier. installation_metadata: PackageInstallationMetadata | None = None ``` +### `installed_version` {#cmem_client.models.package.Package.installed_version} + +```python +installed_version: PackageVersionIdentifier +``` + +The version of the package which is installed. + ### `model_config` {#cmem_client.models.package.Package.model_config} ```python @@ -99,6 +115,11 @@ which were already installed beforehand. ``file``. ``None`` for packages installed before this was tracked. - [**origin_url**](#cmem_client.models.package.PackageInstallationMetadata.origin_url) (str | None) – URL of the marketplace server the package came from. Only set when ``origin_type`` is ``marketplace``. +- [**installed_by**](#cmem_client.models.package.PackageInstallationMetadata.installed_by) (str | None) – Account which installed the package, the ``preferred_username`` +claim of the installing token. ``None`` when the token carries no such claim +and for packages installed before this was recorded. +- [**package_size**](#cmem_client.models.package.PackageInstallationMetadata.package_size) (int | None) – Size of the packaged files in bytes, uncompressed, without the +manifest. ``None`` for packages installed before this was recorded. ### `dependency_level` {#cmem_client.models.package.PackageInstallationMetadata.dependency_level} @@ -112,6 +133,12 @@ dependency_level: int = Field(ge=0, default=0) installed_at: datetime = Field(default=datetime.now(tz=UTC)) ``` +### `installed_by` {#cmem_client.models.package.PackageInstallationMetadata.installed_by} + +```python +installed_by: str | None = None +``` + ### `is_direct_installed` {#cmem_client.models.package.PackageInstallationMetadata.is_direct_installed} ```python @@ -144,6 +171,12 @@ origin_url: str | None = None package_dependency_already_existed: list[str] = Field(default=[]) ``` +### `package_size` {#cmem_client.models.package.PackageInstallationMetadata.package_size} + +```python +package_size: int | None = Field(default=None, ge=0) +``` + ### `python_dependency_already_existed` {#cmem_client.models.package.PackageInstallationMetadata.python_dependency_already_existed} ```python diff --git a/docs/develop/cmem-client-api/api-reference/models/paged_list/index.md b/docs/develop/cmem-client-api/api-reference/models/paged_list/index.md new file mode 100644 index 000000000..af6242cda --- /dev/null +++ b/docs/develop/cmem-client-api/api-reference/models/paged_list/index.md @@ -0,0 +1,66 @@ +--- +title: "cmem-client: models.paged_list module" +description: "Models of paged list responses." +tags: + - API + - Python + - cmem-client +--- + +# `cmem_client.models.paged_list` {#cmem_client.models.paged_list} + + + +Models of paged list responses. + +A paged endpoint of Corporate Memory returns its items one page at a time, together with +a PageDescription of the page. + +**Classes:** + +- [**PageDescription**](#cmem_client.models.paged_list.PageDescription) – A description of a paged list. + +## `PageDescription` {#cmem_client.models.paged_list.PageDescription} + +Bases: [Model](../../models/base/index.md#cmem_client.models.base.Model) + +A description of a paged list. + +**Attributes:** + +- [**size**](#cmem_client.models.paged_list.PageDescription.size) (int) – Number of items requested per page. ``fetch_data()`` stops paging as soon as a page +returns fewer items than this. +- [**number**](#cmem_client.models.paged_list.PageDescription.number) (int) – Zero based index of this page. +- [**total_elements**](#cmem_client.models.paged_list.PageDescription.total_elements) (int) – Total number of items across all pages, sent as ``totalElements``. +- [**total_pages**](#cmem_client.models.paged_list.PageDescription.total_pages) (int) – Total number of pages, sent as ``totalPages``. + +### `model_config` {#cmem_client.models.paged_list.PageDescription.model_config} + +```python +model_config = ConfigDict(extra='allow', populate_by_name=True) +``` + +### `number` {#cmem_client.models.paged_list.PageDescription.number} + +```python +number: int +``` + +### `size` {#cmem_client.models.paged_list.PageDescription.size} + +```python +size: int +``` + +### `total_elements` {#cmem_client.models.paged_list.PageDescription.total_elements} + +```python +total_elements: int = Field(alias='totalElements') +``` + +### `total_pages` {#cmem_client.models.paged_list.PageDescription.total_pages} + +```python +total_pages: int = Field(alias='totalPages') +``` + diff --git a/docs/develop/cmem-client-api/api-reference/models/project/index.md b/docs/develop/cmem-client-api/api-reference/models/project/index.md index 4a2603f65..5f0559c8a 100644 --- a/docs/develop/cmem-client-api/api-reference/models/project/index.md +++ b/docs/develop/cmem-client-api/api-reference/models/project/index.md @@ -25,6 +25,7 @@ project data exchanged with the DataIntegration API. - [**FailedTasksReport**](#cmem_client.models.project.FailedTasksReport) – The failed tasks report - [**Project**](#cmem_client.models.project.Project) – A Build (DataIntegration) Project +- [**ProjectImportStatus**](#cmem_client.models.project.ProjectImportStatus) – Response of the project import status endpoint. - [**ProjectMetaData**](#cmem_client.models.project.ProjectMetaData) – Project Meta Data **Functions:** @@ -146,6 +147,44 @@ Set the label to the name if needed name: str ``` +## `ProjectImportStatus` {#cmem_client.models.project.ProjectImportStatus} + +Bases: [Model](../../models/base/index.md#cmem_client.models.base.Model) + +Response of the project import status endpoint. + +**Attributes:** + +- [**project_id**](#cmem_client.models.project.ProjectImportStatus.project_id) (str) – Identifier of the imported project, sent as ``projectId``. +- [**success**](#cmem_client.models.project.ProjectImportStatus.success) (bool | None) – Whether the import finished successfully. None while the import is still +running, which is what ``import_item()`` polls on. +- [**failure_message**](#cmem_client.models.project.ProjectImportStatus.failure_message) (str | None) – Reason the import failed, sent as ``failureMessage``. Only set when +the import failed. + +### `failure_message` {#cmem_client.models.project.ProjectImportStatus.failure_message} + +```python +failure_message: str | None = Field(alias='failureMessage', default=None) +``` + +### `model_config` {#cmem_client.models.project.ProjectImportStatus.model_config} + +```python +model_config = ConfigDict(extra='allow', populate_by_name=True) +``` + +### `project_id` {#cmem_client.models.project.ProjectImportStatus.project_id} + +```python +project_id: str = Field(alias='projectId') +``` + +### `success` {#cmem_client.models.project.ProjectImportStatus.success} + +```python +success: bool | None = None +``` + ## `ProjectMetaData` {#cmem_client.models.project.ProjectMetaData} Bases: [Model](../../models/base/index.md#cmem_client.models.base.Model) diff --git a/docs/develop/cmem-client-api/api-reference/models/python_package/index.md b/docs/develop/cmem-client-api/api-reference/models/python_package/index.md index 42a05f617..54373405d 100644 --- a/docs/develop/cmem-client-api/api-reference/models/python_package/index.md +++ b/docs/develop/cmem-client-api/api-reference/models/python_package/index.md @@ -19,7 +19,7 @@ keyed by their PyPI name. **Classes:** -- [**PythonPackage**](#cmem_client.models.python_package.PythonPackage) – Installed python package. +- [**PythonPackage**](#cmem_client.models.python_package.PythonPackage) – Installed Python package. **Attributes:** @@ -35,9 +35,9 @@ PipRequirementSpecifier = Annotated[str, Field(title='Pip Requirement Specifier' Bases: [ReadRepositoryItem](../../models/base/index.md#cmem_client.models.base.ReadRepositoryItem) -Installed python package. +Installed Python package. -Represents a python package installed in Corporate Memory +Represents a Python package installed in Corporate Memory. **Attributes:** @@ -72,6 +72,19 @@ model_config = ConfigDict(arbitrary_types_allowed=True, str_strip_whitespace=Tru name: PyPiIdentifier ``` +### `parsed_version` {#cmem_client.models.python_package.PythonPackage.parsed_version} + +```python +parsed_version: Version | None +``` + +The installed version as a comparable version. + +**Returns:** + +- Version | None – The parsed version, or ``None`` if the deployment reports no version or one +- Version | None – which is not a valid Python package version. + ### `version` {#cmem_client.models.python_package.PythonPackage.version} ```python diff --git a/docs/develop/cmem-client-api/api-reference/models/rule_block/index.md b/docs/develop/cmem-client-api/api-reference/models/rule_block/index.md new file mode 100644 index 000000000..146538b4b --- /dev/null +++ b/docs/develop/cmem-client-api/api-reference/models/rule_block/index.md @@ -0,0 +1,320 @@ +--- +title: "cmem-client: models.rule_block module" +description: "Corporate Memory rule block models for data integration." +tags: + - API + - Python + - cmem-client +--- + +# `cmem_client.models.rule_block` {#cmem_client.models.rule_block} + + + +Corporate Memory rule block models for data integration. + +A rule block is a reusable transformation unit: a named operator tree with +declared input ports, referenced from transform and linking rules instead of +being copied into them. Rule blocks are project-scoped task items of type +``RuleBlock`` and are available from Corporate Memory 26.2 onwards. + +The whole body of a rule block is a single opaque object parameter named +``ruleBlockModel``. Only its ports are modelled explicitly here, because they +are the part callers typically read and report; the operator tree, the input +examples and the editor layout are passed through untouched. + +**Classes:** + +- [**RuleBlock**](#cmem_client.models.rule_block.RuleBlock) – A rule block description (Build). +- [**RuleBlockData**](#cmem_client.models.rule_block.RuleBlockData) – The data section of a rule block task. +- [**RuleBlockMetadata**](#cmem_client.models.rule_block.RuleBlockMetadata) – Metadata of a rule block. +- [**RuleBlockModel**](#cmem_client.models.rule_block.RuleBlockModel) – The complete rule block model: ports, operator tree, examples and layout. +- [**RuleBlockPort**](#cmem_client.models.rule_block.RuleBlockPort) – A declared input port of a rule block. +- [**RuleBlockSearchResultSet**](#cmem_client.models.rule_block.RuleBlockSearchResultSet) – A rule block search result set. + +## `RuleBlock` {#cmem_client.models.rule_block.RuleBlock} + +Bases: [ReadRepositoryItem](../../models/base/index.md#cmem_client.models.base.ReadRepositoryItem) + +A rule block description (Build). + +**Functions:** + +- [**get_id**](#cmem_client.models.rule_block.RuleBlock.get_id) – Get the combined ID of the rule block. + +**Attributes:** + +- [**data**](#cmem_client.models.rule_block.RuleBlock.data) ([RuleBlockData](#cmem_client.models.rule_block.RuleBlockData)) – +- [**id**](#cmem_client.models.rule_block.RuleBlock.id) (str) – +- [**item_links**](#cmem_client.models.rule_block.RuleBlock.item_links) (list[[ItemLink](../../models/dataset/index.md#cmem_client.models.dataset.ItemLink)]) – +- [**metadata**](#cmem_client.models.rule_block.RuleBlock.metadata) ([RuleBlockMetadata](#cmem_client.models.rule_block.RuleBlockMetadata)) – +- [**model_config**](#cmem_client.models.rule_block.RuleBlock.model_config) – +- [**project_id**](#cmem_client.models.rule_block.RuleBlock.project_id) (str) – +- [**tags**](#cmem_client.models.rule_block.RuleBlock.tags) (list[[Tag](../../models/common/index.md#cmem_client.models.common.Tag)]) – + +### `data` {#cmem_client.models.rule_block.RuleBlock.data} + +```python +data: RuleBlockData = Field(default_factory=RuleBlockData) +``` + +### `get_id` {#cmem_client.models.rule_block.RuleBlock.get_id} + +```python +get_id() +``` + +Get the combined ID of the rule block. + +### `id` {#cmem_client.models.rule_block.RuleBlock.id} + +```python +id: str +``` + +### `item_links` {#cmem_client.models.rule_block.RuleBlock.item_links} + +```python +item_links: list[ItemLink] = Field(default_factory=list, alias='itemLinks') +``` + +### `metadata` {#cmem_client.models.rule_block.RuleBlock.metadata} + +```python +metadata: RuleBlockMetadata = Field(default_factory=RuleBlockMetadata) +``` + +### `model_config` {#cmem_client.models.rule_block.RuleBlock.model_config} + +```python +model_config = ConfigDict(extra='allow', populate_by_name=True) +``` + +### `project_id` {#cmem_client.models.rule_block.RuleBlock.project_id} + +```python +project_id: str = Field(alias='project', default='') +``` + +### `tags` {#cmem_client.models.rule_block.RuleBlock.tags} + +```python +tags: list[Tag] = Field(default_factory=list) +``` + +## `RuleBlockData` {#cmem_client.models.rule_block.RuleBlockData} + +Bases: [Model](../../models/base/index.md#cmem_client.models.base.Model) + +The data section of a rule block task. + +**Attributes:** + +- [**model_config**](#cmem_client.models.rule_block.RuleBlockData.model_config) – +- [**rule_block_model**](#cmem_client.models.rule_block.RuleBlockData.rule_block_model) ([RuleBlockModel](#cmem_client.models.rule_block.RuleBlockModel)) – +- [**task_type**](#cmem_client.models.rule_block.RuleBlockData.task_type) (str) – +- [**type**](#cmem_client.models.rule_block.RuleBlockData.type) (str) – + +### `model_config` {#cmem_client.models.rule_block.RuleBlockData.model_config} + +```python +model_config = ConfigDict(extra='allow', populate_by_name=True) +``` + +### `rule_block_model` {#cmem_client.models.rule_block.RuleBlockData.rule_block_model} + +```python +rule_block_model: RuleBlockModel = Field(alias='ruleBlockModel', default_factory=RuleBlockModel) +``` + +### `task_type` {#cmem_client.models.rule_block.RuleBlockData.task_type} + +```python +task_type: str = Field(alias='taskType', default='RuleBlock') +``` + +### `type` {#cmem_client.models.rule_block.RuleBlockData.type} + +```python +type: str = 'ruleBlock' +``` + +## `RuleBlockMetadata` {#cmem_client.models.rule_block.RuleBlockMetadata} + +Bases: TypedDict + +Metadata of a rule block. + +**Attributes:** + +- [**created**](#cmem_client.models.rule_block.RuleBlockMetadata.created) (str) – +- [**createdByUser**](#cmem_client.models.rule_block.RuleBlockMetadata.createdByUser) (str) – +- [**description**](#cmem_client.models.rule_block.RuleBlockMetadata.description) (str) – +- [**label**](#cmem_client.models.rule_block.RuleBlockMetadata.label) (str) – +- [**lastModifiedByUser**](#cmem_client.models.rule_block.RuleBlockMetadata.lastModifiedByUser) (str) – +- [**modified**](#cmem_client.models.rule_block.RuleBlockMetadata.modified) (str) – + +### `created` {#cmem_client.models.rule_block.RuleBlockMetadata.created} + +```python +created: str +``` + +### `createdByUser` {#cmem_client.models.rule_block.RuleBlockMetadata.createdByUser} + +```python +createdByUser: str +``` + +### `description` {#cmem_client.models.rule_block.RuleBlockMetadata.description} + +```python +description: str +``` + +### `label` {#cmem_client.models.rule_block.RuleBlockMetadata.label} + +```python +label: str +``` + +### `lastModifiedByUser` {#cmem_client.models.rule_block.RuleBlockMetadata.lastModifiedByUser} + +```python +lastModifiedByUser: str +``` + +### `modified` {#cmem_client.models.rule_block.RuleBlockMetadata.modified} + +```python +modified: str +``` + +## `RuleBlockModel` {#cmem_client.models.rule_block.RuleBlockModel} + +Bases: [Model](../../models/base/index.md#cmem_client.models.base.Model) + +The complete rule block model: ports, operator tree, examples and layout. + +The defaults are those of an empty rule block as DataIntegration creates it; +it rejects a model whose layout lacks ``nodePositions``. + +**Attributes:** + +- [**input_examples**](#cmem_client.models.rule_block.RuleBlockModel.input_examples) (list[Any]) – +- [**layout**](#cmem_client.models.rule_block.RuleBlockModel.layout) (dict[str, Any]) – +- [**model_config**](#cmem_client.models.rule_block.RuleBlockModel.model_config) – +- [**operator_tree**](#cmem_client.models.rule_block.RuleBlockModel.operator_tree) (Any) – +- [**ports**](#cmem_client.models.rule_block.RuleBlockModel.ports) (list[[RuleBlockPort](#cmem_client.models.rule_block.RuleBlockPort)]) – +- [**ui_annotations**](#cmem_client.models.rule_block.RuleBlockModel.ui_annotations) (dict[str, Any]) – + +### `input_examples` {#cmem_client.models.rule_block.RuleBlockModel.input_examples} + +```python +input_examples: list[Any] = Field(alias='inputExamples', default_factory=list) +``` + +### `layout` {#cmem_client.models.rule_block.RuleBlockModel.layout} + +```python +layout: dict[str, Any] = Field(default_factory=lambda: {'nodePositions': {}}) +``` + +### `model_config` {#cmem_client.models.rule_block.RuleBlockModel.model_config} + +```python +model_config = ConfigDict(extra='allow', populate_by_name=True) +``` + +### `operator_tree` {#cmem_client.models.rule_block.RuleBlockModel.operator_tree} + +```python +operator_tree: Any = Field(alias='operatorTree', default=None) +``` + +### `ports` {#cmem_client.models.rule_block.RuleBlockModel.ports} + +```python +ports: list[RuleBlockPort] = Field(default_factory=list) +``` + +### `ui_annotations` {#cmem_client.models.rule_block.RuleBlockModel.ui_annotations} + +```python +ui_annotations: dict[str, Any] = Field(alias='uiAnnotations', default_factory=lambda: {'stickyNotes': []}) +``` + +## `RuleBlockPort` {#cmem_client.models.rule_block.RuleBlockPort} + +Bases: [Model](../../models/base/index.md#cmem_client.models.base.Model) + +A declared input port of a rule block. + +**Attributes:** + +- [**deprecated**](#cmem_client.models.rule_block.RuleBlockPort.deprecated) (bool) – +- [**description**](#cmem_client.models.rule_block.RuleBlockPort.description) (str) – +- [**display_order**](#cmem_client.models.rule_block.RuleBlockPort.display_order) (int) – +- [**id**](#cmem_client.models.rule_block.RuleBlockPort.id) (str) – +- [**label**](#cmem_client.models.rule_block.RuleBlockPort.label) (str) – +- [**model_config**](#cmem_client.models.rule_block.RuleBlockPort.model_config) – + +### `deprecated` {#cmem_client.models.rule_block.RuleBlockPort.deprecated} + +```python +deprecated: bool = False +``` + +### `description` {#cmem_client.models.rule_block.RuleBlockPort.description} + +```python +description: str = '' +``` + +### `display_order` {#cmem_client.models.rule_block.RuleBlockPort.display_order} + +```python +display_order: int = Field(alias='displayOrder', default=0) +``` + +### `id` {#cmem_client.models.rule_block.RuleBlockPort.id} + +```python +id: str = '' +``` + +### `label` {#cmem_client.models.rule_block.RuleBlockPort.label} + +```python +label: str = '' +``` + +### `model_config` {#cmem_client.models.rule_block.RuleBlockPort.model_config} + +```python +model_config = ConfigDict(extra='allow', populate_by_name=True) +``` + +## `RuleBlockSearchResultSet` {#cmem_client.models.rule_block.RuleBlockSearchResultSet} + +Bases: [Model](../../models/base/index.md#cmem_client.models.base.Model) + +A rule block search result set. + +**Attributes:** + +- [**model_config**](#cmem_client.models.rule_block.RuleBlockSearchResultSet.model_config) – +- [**results**](#cmem_client.models.rule_block.RuleBlockSearchResultSet.results) (list[[RuleBlock](#cmem_client.models.rule_block.RuleBlock)]) – + +### `model_config` {#cmem_client.models.rule_block.RuleBlockSearchResultSet.model_config} + +```python +model_config = ConfigDict(extra='allow', populate_by_name=True) +``` + +### `results` {#cmem_client.models.rule_block.RuleBlockSearchResultSet.results} + +```python +results: list[RuleBlock] +``` + diff --git a/docs/develop/cmem-client-api/api-reference/models/status/index.md b/docs/develop/cmem-client-api/api-reference/models/status/index.md index 0389180df..233b72aa0 100644 --- a/docs/develop/cmem-client-api/api-reference/models/status/index.md +++ b/docs/develop/cmem-client-api/api-reference/models/status/index.md @@ -14,14 +14,17 @@ tags: Models for Corporate Memory aggregated status information. Aggregates version and health metadata across the build (DataIntegration), -explore (DataPlatform), shapes catalog, and graph store components. +explore (DataPlatform), shapes catalog, graph store, marketplace, and Graph Insights +components. **Classes:** - [**CmemLicense**](#cmem_client.models.status.CmemLicense) – Corporate Memory license metadata from the DataPlatform info payload. - [**ComponentStatus**](#cmem_client.models.status.ComponentStatus) – Version and health of a single Corporate Memory component. - [**ExploreStatus**](#cmem_client.models.status.ExploreStatus) – Status of the explore (DataPlatform) component, including raw actuator payload. +- [**GraphInsightsStatus**](#cmem_client.models.status.GraphInsightsStatus) – Status of the Graph Insights (semspect) extension. - [**HealthState**](#cmem_client.models.status.HealthState) – Health state of a Corporate Memory component. +- [**MarketplaceStatus**](#cmem_client.models.status.MarketplaceStatus) – Status of the local marketplace bundled beside this Corporate Memory instance. - [**StatusInfo**](#cmem_client.models.status.StatusInfo) – Aggregated status across all Corporate Memory components. - [**StoreInfo**](#cmem_client.models.status.StoreInfo) – Graph store metadata embedded in the DataPlatform info payload. - [**StoreStatus**](#cmem_client.models.status.StoreStatus) – Status of the graph store backing the DataPlatform. @@ -223,6 +226,62 @@ workspaces_to_migrate: list[Any] Workspace IDs flagged by the DataPlatform as requiring configuration migration. +## `GraphInsightsStatus` {#cmem_client.models.status.GraphInsightsStatus} + +Bases: [ComponentStatus](#cmem_client.models.status.ComponentStatus) + +Status of the Graph Insights (semspect) extension. + +**Attributes:** + +- [**applicable**](#cmem_client.models.status.GraphInsightsStatus.applicable) (bool) – Whether Graph Insights is expected on this deployment. +- [**error**](#cmem_client.models.status.GraphInsightsStatus.error) (str | None) – Captured error message if status retrieval failed; None otherwise. +- [**health**](#cmem_client.models.status.GraphInsightsStatus.health) ([HealthState](#cmem_client.models.status.HealthState)) – Health state of the component. +- [**model_config**](#cmem_client.models.status.GraphInsightsStatus.model_config) – +- [**version**](#cmem_client.models.status.GraphInsightsStatus.version) (str) – Component version string as reported by its version/info endpoint. + +### `applicable` {#cmem_client.models.status.GraphInsightsStatus.applicable} + +```python +applicable: bool = True +``` + +Whether Graph Insights is expected on this deployment. + +Graph Insights is an optional extension, so this is set to False when it is not +active for this deployment and user. Not applicable Graph Insights status is left +out of both the overall health rollup and the summary dict. + +### `error` {#cmem_client.models.status.GraphInsightsStatus.error} + +```python +error: str | None = None +``` + +Captured error message if status retrieval failed; None otherwise. + +### `health` {#cmem_client.models.status.GraphInsightsStatus.health} + +```python +health: HealthState = HealthState.UNKNOWN +``` + +Health state of the component. + +### `model_config` {#cmem_client.models.status.GraphInsightsStatus.model_config} + +```python +model_config = ConfigDict(extra='allow', populate_by_name=True) +``` + +### `version` {#cmem_client.models.status.GraphInsightsStatus.version} + +```python +version: str = 'UNKNOWN' +``` + +Component version string as reported by its version/info endpoint. + ## `HealthState` {#cmem_client.models.status.HealthState} Bases: StrEnum @@ -272,6 +331,63 @@ considered DOWN. Missing/empty values become UNKNOWN. - [HealthState](#cmem_client.models.status.HealthState) – The matching HealthState. +## `MarketplaceStatus` {#cmem_client.models.status.MarketplaceStatus} + +Bases: [ComponentStatus](#cmem_client.models.status.ComponentStatus) + +Status of the local marketplace bundled beside this Corporate Memory instance. + +**Attributes:** + +- [**applicable**](#cmem_client.models.status.MarketplaceStatus.applicable) (bool) – Whether the marketplace is expected on this deployment. +- [**error**](#cmem_client.models.status.MarketplaceStatus.error) (str | None) – Captured error message if status retrieval failed; None otherwise. +- [**health**](#cmem_client.models.status.MarketplaceStatus.health) ([HealthState](#cmem_client.models.status.HealthState)) – Health state of the component. +- [**model_config**](#cmem_client.models.status.MarketplaceStatus.model_config) – +- [**version**](#cmem_client.models.status.MarketplaceStatus.version) (str) – Component version string as reported by its version/info endpoint. + +### `applicable` {#cmem_client.models.status.MarketplaceStatus.applicable} + +```python +applicable: bool = True +``` + +Whether the marketplace is expected on this deployment. + +The marketplace was introduced in Corporate Memory v26.2; on an older build it is +optional, so a DOWN marketplace on such a build is set to False here. Not applicable +marketplace status is left out of both the overall health rollup and the summary dict, +instead of being reported as DOWN. + +### `error` {#cmem_client.models.status.MarketplaceStatus.error} + +```python +error: str | None = None +``` + +Captured error message if status retrieval failed; None otherwise. + +### `health` {#cmem_client.models.status.MarketplaceStatus.health} + +```python +health: HealthState = HealthState.UNKNOWN +``` + +Health state of the component. + +### `model_config` {#cmem_client.models.status.MarketplaceStatus.model_config} + +```python +model_config = ConfigDict(extra='allow', populate_by_name=True) +``` + +### `version` {#cmem_client.models.status.MarketplaceStatus.version} + +```python +version: str = 'UNKNOWN' +``` + +Component version string as reported by its version/info endpoint. + ## `SHAPES_CATALOG_VERSION_QUERY` {#cmem_client.models.status.SHAPES_CATALOG_VERSION_QUERY} ```python @@ -294,7 +410,9 @@ Aggregated status across all Corporate Memory components. - [**build**](#cmem_client.models.status.StatusInfo.build) ([ComponentStatus](#cmem_client.models.status.ComponentStatus)) – Status of the build (DataIntegration) component. - [**explore**](#cmem_client.models.status.StatusInfo.explore) ([ExploreStatus](#cmem_client.models.status.ExploreStatus)) – Status of the explore (DataPlatform) component. -- [**health**](#cmem_client.models.status.StatusInfo.health) ([HealthState](#cmem_client.models.status.HealthState)) – Overall health: UP only if every component is UP, DOWN otherwise. +- [**graphinsights**](#cmem_client.models.status.StatusInfo.graphinsights) ([GraphInsightsStatus](#cmem_client.models.status.GraphInsightsStatus)) – Status of the Graph Insights (semspect) extension. +- [**health**](#cmem_client.models.status.StatusInfo.health) ([HealthState](#cmem_client.models.status.HealthState)) – Overall health: UP only if every applicable component is UP, DOWN otherwise. +- [**marketplace**](#cmem_client.models.status.StatusInfo.marketplace) ([MarketplaceStatus](#cmem_client.models.status.MarketplaceStatus)) – Status of the local marketplace bundled beside this Corporate Memory instance. - [**model_config**](#cmem_client.models.status.StatusInfo.model_config) – - [**shapes**](#cmem_client.models.status.StatusInfo.shapes) ([ComponentStatus](#cmem_client.models.status.ComponentStatus)) – Status of the shapes catalog (queried from the explore store). - [**store**](#cmem_client.models.status.StatusInfo.store) ([StoreStatus](#cmem_client.models.status.StoreStatus)) – Status of the graph store backing the DataPlatform. @@ -315,13 +433,32 @@ explore: ExploreStatus = Field(default_factory=ExploreStatus) Status of the explore (DataPlatform) component. +### `graphinsights` {#cmem_client.models.status.StatusInfo.graphinsights} + +```python +graphinsights: GraphInsightsStatus = Field(default_factory=GraphInsightsStatus) +``` + +Status of the Graph Insights (semspect) extension. + ### `health` {#cmem_client.models.status.StatusInfo.health} ```python health: HealthState ``` -Overall health: UP only if every component is UP, DOWN otherwise. +Overall health: UP only if every applicable component is UP, DOWN otherwise. + +Graph Insights only counts once its health is known: a Graph Insights build +without actuator endpoints reports UNKNOWN, which is not treated as a failure. + +### `marketplace` {#cmem_client.models.status.StatusInfo.marketplace} + +```python +marketplace: MarketplaceStatus = Field(default_factory=MarketplaceStatus) +``` + +Status of the local marketplace bundled beside this Corporate Memory instance. ### `model_config` {#cmem_client.models.status.StatusInfo.model_config} diff --git a/docs/develop/cmem-client-api/api-reference/models/variable/index.md b/docs/develop/cmem-client-api/api-reference/models/variable/index.md index e9a45fbea..3222b76a1 100644 --- a/docs/develop/cmem-client-api/api-reference/models/variable/index.md +++ b/docs/develop/cmem-client-api/api-reference/models/variable/index.md @@ -1,6 +1,6 @@ --- title: "cmem-client: models.variable module" -description: "Corporate Memory project variable models for data integration." +description: "Corporate Memory variable models for data integration." tags: - API - Python @@ -11,39 +11,241 @@ tags: -Corporate Memory project variable models for data integration. +Corporate Memory variable models for data integration. -This module defines models for representing project variables within Corporate Memory +This module defines models for representing the variables of Corporate Memory DataIntegration projects. Variables can hold static values or Jinja2 template strings that reference other variables, and are used to parameterize datasets and tasks. +A project variable belongs to a project. An execution variable is declared on a single +task of a project, e.g. a workflow or a dataset. A global variable is defined in the +configuration of DataIntegration, so it belongs to no project and is read-only. + **Classes:** -- [**Variable**](#cmem_client.models.variable.Variable) – A project variable in Corporate Memory DataIntegration. +- [**AllVariablesResponse**](#cmem_client.models.variable.AllVariablesResponse) – The response of the endpoint which lists the global variables and those of all projects. +- [**DependentTask**](#cmem_client.models.variable.DependentTask) – A task which uses a variable, e.g. in a parameter template or an execution variable. +- [**GlobalVariables**](#cmem_client.models.variable.GlobalVariables) – The global variables, which are defined in the configuration of DataIntegration. +- [**ProjectVariables**](#cmem_client.models.variable.ProjectVariables) – A project together with its project variables and the variables of its tasks. +- [**TaskVariables**](#cmem_client.models.variable.TaskVariables) – A task of a project together with its execution variables. +- [**Variable**](#cmem_client.models.variable.Variable) – A project, execution or global variable in Corporate Memory DataIntegration. +- [**VariableDependents**](#cmem_client.models.variable.VariableDependents) – What still uses a variable, so that it cannot be deleted. +- [**VariableError**](#cmem_client.models.variable.VariableError) – An error the server reports for a single variable, e.g. a template it can not resolve. +- [**WorkflowExecutionVariable**](#cmem_client.models.variable.WorkflowExecutionVariable) – An execution variable a workflow run needs. + +## `AllVariablesResponse` {#cmem_client.models.variable.AllVariablesResponse} + +Bases: [Model](../../models/base/index.md#cmem_client.models.base.Model) + +The response of the endpoint which lists the global variables and those of all projects. + +The variables in it carry neither their project nor their task ID, these are given +by the project and task they are listed under. + +**Attributes:** + +- [**global_variables**](#cmem_client.models.variable.AllVariablesResponse.global_variables) ([GlobalVariables](#cmem_client.models.variable.GlobalVariables)) – The global variables. +- [**projects**](#cmem_client.models.variable.AllVariablesResponse.projects) (list[[ProjectVariables](#cmem_client.models.variable.ProjectVariables)]) – The projects the user has access to, with their variables. + +### `global_variables` {#cmem_client.models.variable.AllVariablesResponse.global_variables} + +```python +global_variables: GlobalVariables = Field(alias='global', default_factory=GlobalVariables) +``` + +### `model_config` {#cmem_client.models.variable.AllVariablesResponse.model_config} + +```python +model_config = ConfigDict(extra='allow', populate_by_name=True) +``` + +### `projects` {#cmem_client.models.variable.AllVariablesResponse.projects} + +```python +projects: list[ProjectVariables] = Field(default_factory=list) +``` + +## `DependentTask` {#cmem_client.models.variable.DependentTask} + +Bases: [Model](../../models/base/index.md#cmem_client.models.base.Model) + +A task which uses a variable, e.g. in a parameter template or an execution variable. + +**Attributes:** + +- [**id**](#cmem_client.models.variable.DependentTask.id) (str) – ID of the task. +- [**label**](#cmem_client.models.variable.DependentTask.label) (str) – Label of the task. +- [**task_type**](#cmem_client.models.variable.DependentTask.task_type) (str) – Type of the task, e.g. ``dataset``, ``transform`` or ``workflow``. +Empty where the server does not report it. + +### `id` {#cmem_client.models.variable.DependentTask.id} + +```python +id: str +``` + +### `label` {#cmem_client.models.variable.DependentTask.label} + +```python +label: str = '' +``` + +### `model_config` {#cmem_client.models.variable.DependentTask.model_config} + +```python +model_config = ConfigDict(extra='allow', populate_by_name=True) +``` + +### `task_type` {#cmem_client.models.variable.DependentTask.task_type} + +```python +task_type: str = Field(alias='taskType', default='') +``` + +## `GlobalVariables` {#cmem_client.models.variable.GlobalVariables} + +Bases: [Model](../../models/base/index.md#cmem_client.models.base.Model) + +The global variables, which are defined in the configuration of DataIntegration. + +**Attributes:** + +- [**variables**](#cmem_client.models.variable.GlobalVariables.variables) (list[[Variable](#cmem_client.models.variable.Variable)]) – The global variables. + +### `model_config` {#cmem_client.models.variable.GlobalVariables.model_config} + +```python +model_config = ConfigDict(extra='allow', populate_by_name=True) +``` + +### `variables` {#cmem_client.models.variable.GlobalVariables.variables} + +```python +variables: list[Variable] = Field(default_factory=list) +``` + +## `ProjectVariables` {#cmem_client.models.variable.ProjectVariables} + +Bases: [Model](../../models/base/index.md#cmem_client.models.base.Model) + +A project together with its project variables and the variables of its tasks. + +**Attributes:** + +- [**id**](#cmem_client.models.variable.ProjectVariables.id) (str) – ID of the project. +- [**label**](#cmem_client.models.variable.ProjectVariables.label) (str) – Label of the project. +- [**variables**](#cmem_client.models.variable.ProjectVariables.variables) (list[[Variable](#cmem_client.models.variable.Variable)]) – The project variables. +- [**errors**](#cmem_client.models.variable.ProjectVariables.errors) (list[[VariableError](#cmem_client.models.variable.VariableError)]) – Errors of single project variables. +- [**tasks**](#cmem_client.models.variable.ProjectVariables.tasks) (list[[TaskVariables](#cmem_client.models.variable.TaskVariables)]) – All tasks of the project, including tasks without execution variables. + +### `errors` {#cmem_client.models.variable.ProjectVariables.errors} + +```python +errors: list[VariableError] = Field(default_factory=list) +``` + +### `id` {#cmem_client.models.variable.ProjectVariables.id} + +```python +id: str +``` + +### `label` {#cmem_client.models.variable.ProjectVariables.label} + +```python +label: str = '' +``` + +### `model_config` {#cmem_client.models.variable.ProjectVariables.model_config} + +```python +model_config = ConfigDict(extra='allow', populate_by_name=True) +``` + +### `tasks` {#cmem_client.models.variable.ProjectVariables.tasks} + +```python +tasks: list[TaskVariables] = Field(default_factory=list) +``` + +### `variables` {#cmem_client.models.variable.ProjectVariables.variables} + +```python +variables: list[Variable] = Field(default_factory=list) +``` + +## `TaskVariables` {#cmem_client.models.variable.TaskVariables} + +Bases: [Model](../../models/base/index.md#cmem_client.models.base.Model) + +A task of a project together with its execution variables. + +**Attributes:** + +- [**id**](#cmem_client.models.variable.TaskVariables.id) (str) – ID of the task. +- [**label**](#cmem_client.models.variable.TaskVariables.label) (str) – Label of the task. +- [**variables**](#cmem_client.models.variable.TaskVariables.variables) (list[[Variable](#cmem_client.models.variable.Variable)]) – The execution variables declared on the task. +- [**errors**](#cmem_client.models.variable.TaskVariables.errors) (list[[VariableError](#cmem_client.models.variable.VariableError)]) – Errors of single execution variables of the task. + +### `errors` {#cmem_client.models.variable.TaskVariables.errors} + +```python +errors: list[VariableError] = Field(default_factory=list) +``` + +### `id` {#cmem_client.models.variable.TaskVariables.id} + +```python +id: str +``` + +### `label` {#cmem_client.models.variable.TaskVariables.label} + +```python +label: str = '' +``` + +### `model_config` {#cmem_client.models.variable.TaskVariables.model_config} + +```python +model_config = ConfigDict(extra='allow', populate_by_name=True) +``` + +### `variables` {#cmem_client.models.variable.TaskVariables.variables} + +```python +variables: list[Variable] = Field(default_factory=list) +``` ## `Variable` {#cmem_client.models.variable.Variable} Bases: [ReadRepositoryItem](../../models/base/index.md#cmem_client.models.base.ReadRepositoryItem) -A project variable in Corporate Memory DataIntegration. +A project, execution or global variable in Corporate Memory DataIntegration. **Attributes:** -- [**name**](#cmem_client.models.variable.Variable.name) (str) – Name of the variable, unique within its project. This is what a template -refers to. -- [**project_id**](#cmem_client.models.variable.Variable.project_id) (str) – ID of the project the variable belongs to. +- [**name**](#cmem_client.models.variable.Variable.name) (str) – Name of the variable, unique within its project or task. This is what a +template refers to. +- [**project_id**](#cmem_client.models.variable.Variable.project_id) (str) – ID of the project the variable belongs to. Together with ``name``, +and ``task_id`` for an execution variable, it forms the key of the repository. +Empty for a global variable. +- [**task_id**](#cmem_client.models.variable.Variable.task_id) (str | None) – ID of the task an execution variable is declared on. ``None`` for a +project variable. - [**value**](#cmem_client.models.variable.Variable.value) (str) – Resolved value of the variable. For a templated variable this is the -rendered result, so it is read-only. +rendered result, so it is read-only. Empty where the server masks the value, +which it does for sensitive variables when listing all variables. - [**template**](#cmem_client.models.variable.Variable.template) (str) – Jinja2 template the value is rendered from. Empty for a variable which holds a static value. - [**description**](#cmem_client.models.variable.Variable.description) (str) – Description of the variable as maintained in the project. - [**is_sensitive**](#cmem_client.models.variable.Variable.is_sensitive) (bool) – Whether the value is treated as a secret. Sensitive values are masked by the user interface. -- [**scope**](#cmem_client.models.variable.Variable.scope) (str) – Scope the variable is defined in, ``project`` for a project variable. +- [**scope**](#cmem_client.models.variable.Variable.scope) (str) – Scope the variable is defined in, ``project`` for a project variable, +``execution`` for an execution variable and ``global`` for a global variable. **Functions:** -- [**get_id**](#cmem_client.models.variable.Variable.get_id) – Get the combined ID of the variable in the form ``project_id:name``. +- [**get_id**](#cmem_client.models.variable.Variable.get_id) – Get the combined ID of the variable. ### `description` {#cmem_client.models.variable.Variable.description} @@ -57,7 +259,12 @@ description: str = '' get_id() ``` -Get the combined ID of the variable in the form ``project_id:name``. +Get the combined ID of the variable. + +**Returns:** + +- str – ``project_id:name`` for a project variable, ``project_id:task_id:name`` +- str – for an execution variable and ``name`` for a global variable. ### `is_sensitive` {#cmem_client.models.variable.Variable.is_sensitive} @@ -80,7 +287,7 @@ name: str ### `project_id` {#cmem_client.models.variable.Variable.project_id} ```python -project_id: str = Field(alias='project') +project_id: str = Field(alias='project', default='') ``` ### `scope` {#cmem_client.models.variable.Variable.scope} @@ -89,6 +296,12 @@ project_id: str = Field(alias='project') scope: str = 'project' ``` +### `task_id` {#cmem_client.models.variable.Variable.task_id} + +```python +task_id: str | None = Field(alias='task', default=None) +``` + ### `template` {#cmem_client.models.variable.Variable.template} ```python @@ -101,3 +314,117 @@ template: str = '' value: str = '' ``` +## `VariableDependents` {#cmem_client.models.variable.VariableDependents} + +Bases: [Model](../../models/base/index.md#cmem_client.models.base.Model) + +What still uses a variable, so that it cannot be deleted. + +**Attributes:** + +- [**variables**](#cmem_client.models.variable.VariableDependents.variables) (list[str]) – Names of the variables in the same scope whose templates use the +variable: project variables for a project variable, execution variables of +the same task for an execution variable. +- [**tasks**](#cmem_client.models.variable.VariableDependents.tasks) (list[[DependentTask](#cmem_client.models.variable.DependentTask)]) – Tasks which use the variable. + +### `model_config` {#cmem_client.models.variable.VariableDependents.model_config} + +```python +model_config = ConfigDict(extra='allow', populate_by_name=True) +``` + +### `tasks` {#cmem_client.models.variable.VariableDependents.tasks} + +```python +tasks: list[DependentTask] = Field(alias='dependentTasks', default_factory=list) +``` + +### `variables` {#cmem_client.models.variable.VariableDependents.variables} + +```python +variables: list[str] = Field(alias='dependentVariables', default_factory=list) +``` + +## `VariableError` {#cmem_client.models.variable.VariableError} + +Bases: [Model](../../models/base/index.md#cmem_client.models.base.Model) + +An error the server reports for a single variable, e.g. a template it can not resolve. + +**Attributes:** + +- [**variable_name**](#cmem_client.models.variable.VariableError.variable_name) (str) – Name of the variable the error belongs to. +- [**message**](#cmem_client.models.variable.VariableError.message) (str) – The error message. + +### `message` {#cmem_client.models.variable.VariableError.message} + +```python +message: str = '' +``` + +### `model_config` {#cmem_client.models.variable.VariableError.model_config} + +```python +model_config = ConfigDict(extra='allow', populate_by_name=True) +``` + +### `variable_name` {#cmem_client.models.variable.VariableError.variable_name} + +```python +variable_name: str = Field(alias='variableName') +``` + +## `WorkflowExecutionVariable` {#cmem_client.models.variable.WorkflowExecutionVariable} + +Bases: [Model](../../models/base/index.md#cmem_client.models.base.Model) + +An execution variable a workflow run needs. + +**Attributes:** + +- [**name**](#cmem_client.models.variable.WorkflowExecutionVariable.name) (str) – Name of the variable, referenced as ``{{execution.}}``. +- [**required**](#cmem_client.models.variable.WorkflowExecutionVariable.required) (bool) – Whether the variable has to be provided when the run is started, +because it is used at execution time but neither defined on the workflow +nor set by a preceding node. +- [**default**](#cmem_client.models.variable.WorkflowExecutionVariable.default) ([Variable](#cmem_client.models.variable.Variable) | None) – The default defined on the workflow, if any. Only the defaults of the +executed workflow apply to a run, defaults of sub-tasks do not. The value +of a sensitive default is omitted by the server. +- [**referenced_by**](#cmem_client.models.variable.WorkflowExecutionVariable.referenced_by) (list[[DependentTask](#cmem_client.models.variable.DependentTask)]) – Tasks which use the variable at execution time. +- [**set_by**](#cmem_client.models.variable.WorkflowExecutionVariable.set_by) (list[[DependentTask](#cmem_client.models.variable.DependentTask)]) – Tasks which set the variable during the run. + +### `default` {#cmem_client.models.variable.WorkflowExecutionVariable.default} + +```python +default: Variable | None = None +``` + +### `model_config` {#cmem_client.models.variable.WorkflowExecutionVariable.model_config} + +```python +model_config = ConfigDict(extra='allow', populate_by_name=True) +``` + +### `name` {#cmem_client.models.variable.WorkflowExecutionVariable.name} + +```python +name: str +``` + +### `referenced_by` {#cmem_client.models.variable.WorkflowExecutionVariable.referenced_by} + +```python +referenced_by: list[DependentTask] = Field(alias='referencedBy', default_factory=list) +``` + +### `required` {#cmem_client.models.variable.WorkflowExecutionVariable.required} + +```python +required: bool = False +``` + +### `set_by` {#cmem_client.models.variable.WorkflowExecutionVariable.set_by} + +```python +set_by: list[DependentTask] = Field(alias='setBy', default_factory=list) +``` + diff --git a/docs/develop/cmem-client-api/api-reference/models/workflow/index.md b/docs/develop/cmem-client-api/api-reference/models/workflow/index.md index 957bdc348..9f9c9188f 100644 --- a/docs/develop/cmem-client-api/api-reference/models/workflow/index.md +++ b/docs/develop/cmem-client-api/api-reference/models/workflow/index.md @@ -29,6 +29,10 @@ and report its own status instead of going back through the repository: **Classes:** - [**Workflow**](#cmem_client.models.workflow.Workflow) – A workflow +- [**WorkflowReport**](#cmem_client.models.workflow.WorkflowReport) – One workflow execution report. +- [**WorkflowReportFinishStatus**](#cmem_client.models.workflow.WorkflowReportFinishStatus) – Final execution status recorded in a workflow report. +- [**WorkflowReportInfo**](#cmem_client.models.workflow.WorkflowReportInfo) – One entry of a workflow execution report list. +- [**WorkflowReportMetaData**](#cmem_client.models.workflow.WorkflowReportMetaData) – Metadata section of a workflow execution report. - [**WorkflowSearchResultSet**](#cmem_client.models.workflow.WorkflowSearchResultSet) – Wrapper for the search API response envelope. - [**WorkflowStatus**](#cmem_client.models.workflow.WorkflowStatus) – Workflow execution status @@ -87,10 +91,11 @@ forms the ``{project_id}:{id}`` key of the repository. which is what ``execute_io()`` writes its payload into. - [**variable_outputs**](#cmem_client.models.workflow.Workflow.variable_outputs) (list[str]) – IDs of the outputs which can be replaced at execution time, which is what ``execute_io()`` reads its result from. -- [**warnings**](#cmem_client.models.workflow.Workflow.warnings) (list[str]) – Warnings DataIntegration reported for the workflow. +- [**warnings**](#cmem_client.models.workflow.Workflow.warnings) (list[str]) – Warnings DataIntegration reported for the workflow. DataIntegration +reports them as plain strings or as objects with a ``message``. - [**tags**](#cmem_client.models.workflow.Workflow.tags) (list[[Tag](../../models/common/index.md#cmem_client.models.common.Tag)]) – Tags attached to the workflow. - [**parameters**](#cmem_client.models.workflow.Workflow.parameters) (dict) – Raw parameters as returned by the search endpoint. Excluded from -serialization; the variable inputs and outputs are read out of it. +serialization. **Functions:** @@ -200,6 +205,228 @@ variable_outputs: list[str] = Field(alias='variableOutputs', default_factory=lis warnings: list[str] = Field(default_factory=list) ``` +## `WorkflowReport` {#cmem_client.models.workflow.WorkflowReport} + +Bases: [Model](../../models/base/index.md#cmem_client.models.base.Model) + +One workflow execution report. + +**Attributes:** + +- [**meta_data**](#cmem_client.models.workflow.WorkflowReport.meta_data) ([WorkflowReportMetaData](#cmem_client.models.workflow.WorkflowReportMetaData)) – Timing and final status of the execution. +- [**value**](#cmem_client.models.workflow.WorkflowReport.value) (dict[str, Any]) – Report body describing the executed operation (warnings, error, entity +counts, and nested reports of sub-tasks). Kept as a plain dictionary +because its shape depends on the type of task that ran. + +### `meta_data` {#cmem_client.models.workflow.WorkflowReport.meta_data} + +```python +meta_data: WorkflowReportMetaData = Field(alias='metaData') +``` + +### `model_config` {#cmem_client.models.workflow.WorkflowReport.model_config} + +```python +model_config = ConfigDict(extra='allow', populate_by_name=True) +``` + +### `value` {#cmem_client.models.workflow.WorkflowReport.value} + +```python +value: dict[str, Any] = Field(default_factory=dict) +``` + +## `WorkflowReportFinishStatus` {#cmem_client.models.workflow.WorkflowReportFinishStatus} + +Bases: [Model](../../models/base/index.md#cmem_client.models.base.Model) + +Final execution status recorded in a workflow report. + +**Attributes:** + +- [**status_name**](#cmem_client.models.workflow.WorkflowReportFinishStatus.status_name) (str) – Coarse state of the execution, e.g. ``Finished``. +- [**concrete_status**](#cmem_client.models.workflow.WorkflowReportFinishStatus.concrete_status) (str) – What the state amounts to, e.g. ``Successful``, ``Failed`` or +``Cancelled``. +- [**is_running**](#cmem_client.models.workflow.WorkflowReportFinishStatus.is_running) (bool) – Whether the execution was still going when the report was written. +Always ``False`` for a finished report. +- [**progress**](#cmem_client.models.workflow.WorkflowReportFinishStatus.progress) (float | None) – How far the execution got, in percent. +- [**message**](#cmem_client.models.workflow.WorkflowReportFinishStatus.message) (str) – Human readable status message. +- [**failed**](#cmem_client.models.workflow.WorkflowReportFinishStatus.failed) (bool) – Whether the execution failed. +- [**last_update_time**](#cmem_client.models.workflow.WorkflowReportFinishStatus.last_update_time) (int) – When the status was last updated, as a Unix timestamp in +milliseconds. +- [**runtime**](#cmem_client.models.workflow.WorkflowReportFinishStatus.runtime) (int | None) – How long the execution took, in milliseconds. +- [**cancelled**](#cmem_client.models.workflow.WorkflowReportFinishStatus.cancelled) (bool | None) – Whether the execution was cancelled. +- [**exception_message**](#cmem_client.models.workflow.WorkflowReportFinishStatus.exception_message) (str | None) – Message of the exception which ended the execution, if one +did. + +### `cancelled` {#cmem_client.models.workflow.WorkflowReportFinishStatus.cancelled} + +```python +cancelled: bool | None = None +``` + +### `concrete_status` {#cmem_client.models.workflow.WorkflowReportFinishStatus.concrete_status} + +```python +concrete_status: str = Field(alias='concreteStatus') +``` + +### `exception_message` {#cmem_client.models.workflow.WorkflowReportFinishStatus.exception_message} + +```python +exception_message: str | None = Field(default=None, alias='exceptionMessage') +``` + +### `failed` {#cmem_client.models.workflow.WorkflowReportFinishStatus.failed} + +```python +failed: bool +``` + +### `is_running` {#cmem_client.models.workflow.WorkflowReportFinishStatus.is_running} + +```python +is_running: bool = Field(alias='isRunning') +``` + +### `last_update_time` {#cmem_client.models.workflow.WorkflowReportFinishStatus.last_update_time} + +```python +last_update_time: int = Field(alias='lastUpdateTime') +``` + +### `message` {#cmem_client.models.workflow.WorkflowReportFinishStatus.message} + +```python +message: str +``` + +### `model_config` {#cmem_client.models.workflow.WorkflowReportFinishStatus.model_config} + +```python +model_config = ConfigDict(extra='allow', populate_by_name=True) +``` + +### `progress` {#cmem_client.models.workflow.WorkflowReportFinishStatus.progress} + +```python +progress: float | None = None +``` + +### `runtime` {#cmem_client.models.workflow.WorkflowReportFinishStatus.runtime} + +```python +runtime: int | None = None +``` + +### `status_name` {#cmem_client.models.workflow.WorkflowReportFinishStatus.status_name} + +```python +status_name: str = Field(alias='statusName') +``` + +## `WorkflowReportInfo` {#cmem_client.models.workflow.WorkflowReportInfo} + +Bases: [Model](../../models/base/index.md#cmem_client.models.base.Model) + +One entry of a workflow execution report list. + +**Attributes:** + +- [**project**](#cmem_client.models.workflow.WorkflowReportInfo.project) (str) – ID of the project holding the workflow. +- [**task**](#cmem_client.models.workflow.WorkflowReportInfo.task) (str) – ID of the workflow task. +- [**time**](#cmem_client.models.workflow.WorkflowReportInfo.time) (datetime) – Timestamp identifying the report, passed to +[get_report](../../repositories/workflows/index.md#cmem_client.repositories.workflows.WorkflowsRepository.get_report) +to retrieve it. + +### `model_config` {#cmem_client.models.workflow.WorkflowReportInfo.model_config} + +```python +model_config = ConfigDict(extra='allow', populate_by_name=True) +``` + +### `project` {#cmem_client.models.workflow.WorkflowReportInfo.project} + +```python +project: str +``` + +### `task` {#cmem_client.models.workflow.WorkflowReportInfo.task} + +```python +task: str +``` + +### `time` {#cmem_client.models.workflow.WorkflowReportInfo.time} + +```python +time: datetime +``` + +## `WorkflowReportMetaData` {#cmem_client.models.workflow.WorkflowReportMetaData} + +Bases: [Model](../../models/base/index.md#cmem_client.models.base.Model) + +Metadata section of a workflow execution report. + +**Attributes:** + +- [**started_by_user**](#cmem_client.models.workflow.WorkflowReportMetaData.started_by_user) (str | None) – Account name of the user who started the execution, if known. +- [**queued_at**](#cmem_client.models.workflow.WorkflowReportMetaData.queued_at) (datetime | None) – When the execution was queued. +- [**started_at**](#cmem_client.models.workflow.WorkflowReportMetaData.started_at) (datetime | None) – When the execution actually started. +- [**finished_at**](#cmem_client.models.workflow.WorkflowReportMetaData.finished_at) (datetime | None) – When the execution finished. +- [**cancelled_at**](#cmem_client.models.workflow.WorkflowReportMetaData.cancelled_at) (datetime | None) – When the execution was cancelled, if it was. +- [**cancelled_by**](#cmem_client.models.workflow.WorkflowReportMetaData.cancelled_by) (str | None) – Account name of the user who cancelled the execution, if it was. +- [**finish_status**](#cmem_client.models.workflow.WorkflowReportMetaData.finish_status) ([WorkflowReportFinishStatus](#cmem_client.models.workflow.WorkflowReportFinishStatus)) – The execution's final status. + +### `cancelled_at` {#cmem_client.models.workflow.WorkflowReportMetaData.cancelled_at} + +```python +cancelled_at: datetime | None = Field(default=None, alias='cancelledAt') +``` + +### `cancelled_by` {#cmem_client.models.workflow.WorkflowReportMetaData.cancelled_by} + +```python +cancelled_by: str | None = Field(default=None, alias='cancelledBy') +``` + +### `finish_status` {#cmem_client.models.workflow.WorkflowReportMetaData.finish_status} + +```python +finish_status: WorkflowReportFinishStatus = Field(alias='finishStatus') +``` + +### `finished_at` {#cmem_client.models.workflow.WorkflowReportMetaData.finished_at} + +```python +finished_at: datetime | None = Field(default=None, alias='finishedAt') +``` + +### `model_config` {#cmem_client.models.workflow.WorkflowReportMetaData.model_config} + +```python +model_config = ConfigDict(extra='allow', populate_by_name=True) +``` + +### `queued_at` {#cmem_client.models.workflow.WorkflowReportMetaData.queued_at} + +```python +queued_at: datetime | None = Field(default=None, alias='queuedAt') +``` + +### `started_at` {#cmem_client.models.workflow.WorkflowReportMetaData.started_at} + +```python +started_at: datetime | None = Field(default=None, alias='startedAt') +``` + +### `started_by_user` {#cmem_client.models.workflow.WorkflowReportMetaData.started_by_user} + +```python +started_by_user: str | None = Field(default=None, alias='startedByUser') +``` + ## `WorkflowSearchResultSet` {#cmem_client.models.workflow.WorkflowSearchResultSet} Bases: [Model](../../models/base/index.md#cmem_client.models.base.Model) diff --git a/docs/develop/cmem-client-api/api-reference/repositories/.pages b/docs/develop/cmem-client-api/api-reference/repositories/.pages index 57601e94d..4346ecbfc 100644 --- a/docs/develop/cmem-client-api/api-reference/repositories/.pages +++ b/docs/develop/cmem-client-api/api-reference/repositories/.pages @@ -6,12 +6,13 @@ nav: - datasets: datasets - files: files - graph_imports: graph_imports - - graph_insights: graph_insights + - graph_snapshots: graph_snapshots - graphs: graphs - marketplace_packages: marketplace_packages - projects: projects - python_packages: python_packages - queries: queries + - rule_blocks: rule_blocks - schedulers: schedulers - user_accounts: user_accounts - validations: validations diff --git a/docs/develop/cmem-client-api/api-reference/repositories/base/paged_list/index.md b/docs/develop/cmem-client-api/api-reference/repositories/base/paged_list/index.md index 4e412933f..581034ea3 100644 --- a/docs/develop/cmem-client-api/api-reference/repositories/base/paged_list/index.md +++ b/docs/develop/cmem-client-api/api-reference/repositories/base/paged_list/index.md @@ -23,53 +23,8 @@ in a paginated format with metadata about page size, number, and totals. **Classes:** -- [**PageDescription**](#cmem_client.repositories.base.paged_list.PageDescription) – A description of a paged list. - [**PagedListRepository**](#cmem_client.repositories.base.paged_list.PagedListRepository) – Repository that uses a paged list endpoint. -## `PageDescription` {#cmem_client.repositories.base.paged_list.PageDescription} - -Bases: [Model](../../../models/base/index.md#cmem_client.models.base.Model) - -A description of a paged list. - -**Attributes:** - -- [**size**](#cmem_client.repositories.base.paged_list.PageDescription.size) (int) – Number of items requested per page. ``fetch_data()`` stops paging as soon as a page -returns fewer items than this. -- [**number**](#cmem_client.repositories.base.paged_list.PageDescription.number) (int) – Zero based index of this page. -- [**total_elements**](#cmem_client.repositories.base.paged_list.PageDescription.total_elements) (int) – Total number of items across all pages, sent as ``totalElements``. -- [**total_pages**](#cmem_client.repositories.base.paged_list.PageDescription.total_pages) (int) – Total number of pages, sent as ``totalPages``. - -### `model_config` {#cmem_client.repositories.base.paged_list.PageDescription.model_config} - -```python -model_config = ConfigDict(extra='allow', populate_by_name=True) -``` - -### `number` {#cmem_client.repositories.base.paged_list.PageDescription.number} - -```python -number: int -``` - -### `size` {#cmem_client.repositories.base.paged_list.PageDescription.size} - -```python -size: int -``` - -### `total_elements` {#cmem_client.repositories.base.paged_list.PageDescription.total_elements} - -```python -total_elements: int = Field(alias='totalElements') -``` - -### `total_pages` {#cmem_client.repositories.base.paged_list.PageDescription.total_pages} - -```python -total_pages: int = Field(alias='totalPages') -``` - ## `PagedListRepository` {#cmem_client.repositories.base.paged_list.PagedListRepository} Bases: [Repository](../../../repositories/base/abc/index.md#cmem_client.repositories.base.abc.Repository) diff --git a/docs/develop/cmem-client-api/api-reference/repositories/base/plain_list/index.md b/docs/develop/cmem-client-api/api-reference/repositories/base/plain_list/index.md index 7e5edcdbb..7ebe88af5 100644 --- a/docs/develop/cmem-client-api/api-reference/repositories/base/plain_list/index.md +++ b/docs/develop/cmem-client-api/api-reference/repositories/base/plain_list/index.md @@ -15,7 +15,7 @@ Repository implementation for simple list API endpoints. This module provides PlainListRepository, a repository implementation for API endpoints that return a simple array of objects without pagination. -It's commonly used with Corporate Memory's DataIntegration (build) APIs +It is commonly used with Corporate Memory's DataIntegration (build) APIs that provide straightforward list responses. The PlainListRepository fetches the entire list in a single request and diff --git a/docs/develop/cmem-client-api/api-reference/repositories/files/index.md b/docs/develop/cmem-client-api/api-reference/repositories/files/index.md index dd786b38d..d756f6020 100644 --- a/docs/develop/cmem-client-api/api-reference/repositories/files/index.md +++ b/docs/develop/cmem-client-api/api-reference/repositories/files/index.md @@ -171,7 +171,7 @@ Export an item from the repository to a file path. **Raises:** - [RepositoryItemNotFoundError](../../exceptions/index.md#cmem_client.exceptions.RepositoryItemNotFoundError) – If the specified item key is not found. -- [RepositoryReadError](../../exceptions/index.md#cmem_client.exceptions.RepositoryReadError) – If there's an error during export or path mismatch. +- [RepositoryReadError](../../exceptions/index.md#cmem_client.exceptions.RepositoryReadError) – If there is an error during export or path mismatch. ### `fetch_data` {#cmem_client.repositories.files.FilesRepository.fetch_data} diff --git a/docs/develop/cmem-client-api/api-reference/repositories/graph_insights/index.md b/docs/develop/cmem-client-api/api-reference/repositories/graph_snapshots/index.md similarity index 50% rename from docs/develop/cmem-client-api/api-reference/repositories/graph_insights/index.md rename to docs/develop/cmem-client-api/api-reference/repositories/graph_snapshots/index.md index 6ab8f9a55..a2d8082c2 100644 --- a/docs/develop/cmem-client-api/api-reference/repositories/graph_insights/index.md +++ b/docs/develop/cmem-client-api/api-reference/repositories/graph_snapshots/index.md @@ -1,5 +1,5 @@ --- -title: "cmem-client: repositories.graph_insights module" +title: "cmem-client: repositories.graph_snapshots module" description: "Repository for the Graph Insights snapshots of Corporate Memory." tags: - API @@ -7,15 +7,15 @@ tags: - cmem-client --- -# `cmem_client.repositories.graph_insights` {#cmem_client.repositories.graph_insights} +# `cmem_client.repositories.graph_snapshots` {#cmem_client.repositories.graph_snapshots} Repository for the Graph Insights snapshots of Corporate Memory. -Provides GraphInsightsRepository for creating statistics snapshots of a graph, polling +Provides GraphSnapshotsRepository for creating statistics snapshots of a graph, polling their computation and deleting them. Graph Insights is an optional extension, so check -that it is enabled before using it. +that it is available with `client.graphinsights.is_available()` before using it. **Examples:** @@ -24,26 +24,26 @@ Check whether the extension is available, then create a snapshot: ```pycon >>> from cmem_client.client import Client >>> client = Client.from_env() ->>> client.graph_insights.is_available() ->>> snapshot_id = client.graph_insights.create("https://ns.eccenca.com/data/config/") ->>> client.graph_insights.wait_for_completion(snapshot_id) +>>> client.graphinsights.is_available() +>>> snapshot_id = client.graph_snapshots.create("https://ns.eccenca.com/data/config/") +>>> client.graph_snapshots.wait_for_completion(snapshot_id) ``` Read the snapshots and drop them again: ```pycon ->>> client.graph_insights.fetch_data() ->>> list(client.graph_insights) ->>> client.graph_insights.delete_item(snapshot_id) +>>> client.graph_snapshots.fetch_data() +>>> list(client.graph_snapshots) +>>> client.graph_snapshots.delete_item(snapshot_id) ``` **Classes:** -- [**GraphInsightDeleteConfig**](#cmem_client.repositories.graph_insights.GraphInsightDeleteConfig) – Graph Insight Snapshot Delete Configuration. -- [**GraphInsightUpdateConfig**](#cmem_client.repositories.graph_insights.GraphInsightUpdateConfig) – Graph Insight Snapshot Update Configuration. -- [**GraphInsightsRepository**](#cmem_client.repositories.graph_insights.GraphInsightsRepository) – Repository for the semspect Graph Insights extension. +- [**GraphSnapshotDeleteConfig**](#cmem_client.repositories.graph_snapshots.GraphSnapshotDeleteConfig) – Graph Insight Snapshot Delete Configuration. +- [**GraphSnapshotUpdateConfig**](#cmem_client.repositories.graph_snapshots.GraphSnapshotUpdateConfig) – Graph Insight Snapshot Update Configuration. +- [**GraphSnapshotsRepository**](#cmem_client.repositories.graph_snapshots.GraphSnapshotsRepository) – Repository for the snapshots of the semspect Graph Insights extension. -## `GraphInsightDeleteConfig` {#cmem_client.repositories.graph_insights.GraphInsightDeleteConfig} +## `GraphSnapshotDeleteConfig` {#cmem_client.repositories.graph_snapshots.GraphSnapshotDeleteConfig} Bases: [DeleteConfig](../../repositories/protocols/delete_item/index.md#cmem_client.repositories.protocols.delete_item.DeleteConfig) @@ -53,7 +53,7 @@ Graph Insight Snapshot Delete Configuration. - **model_config** – -## `GraphInsightUpdateConfig` {#cmem_client.repositories.graph_insights.GraphInsightUpdateConfig} +## `GraphSnapshotUpdateConfig` {#cmem_client.repositories.graph_snapshots.GraphSnapshotUpdateConfig} Bases: [UpdateConfig](../../repositories/protocols/update_item/index.md#cmem_client.repositories.protocols.update_item.UpdateConfig) @@ -63,38 +63,39 @@ Graph Insight Snapshot Update Configuration. - **model_config** – -## `GraphInsightsRepository` {#cmem_client.repositories.graph_insights.GraphInsightsRepository} +## `GraphSnapshotsRepository` {#cmem_client.repositories.graph_snapshots.GraphSnapshotsRepository} ```python -GraphInsightsRepository(client) +GraphSnapshotsRepository(client) ``` Bases: [Repository](../../repositories/base/abc/index.md#cmem_client.repositories.base.abc.Repository)[[GraphInsightSnapshot](../../models/graph_insight/index.md#cmem_client.models.graph_insight.GraphInsightSnapshot)], [DeleteItemProtocol](../../repositories/protocols/delete_item/index.md#cmem_client.repositories.protocols.delete_item.DeleteItemProtocol), [UpdateItemProtocol](../../repositories/protocols/update_item/index.md#cmem_client.repositories.protocols.update_item.UpdateItemProtocol) -Repository for the semspect Graph Insights extension. +Repository for the snapshots of the semspect Graph Insights extension. Does not auto-fetch on init because semspect is optional and may not be installed. -Call fetch_data() explicitly before iterating snapshots. +Check `client.graphinsights.is_available()`, and call fetch_data() explicitly before +iterating snapshots. **Functions:** -- [**create**](#cmem_client.repositories.graph_insights.GraphInsightsRepository.create) – Create or update a snapshot for the given graph IRI. -- [**delete_all**](#cmem_client.repositories.graph_insights.GraphInsightsRepository.delete_all) – Delete all snapshots via the bulk DELETE endpoint. -- [**delete_item**](#cmem_client.repositories.graph_insights.GraphInsightsRepository.delete_item) – Delete an item from the repository -- [**fetch_data**](#cmem_client.repositories.graph_insights.GraphInsightsRepository.fetch_data) – Fetch all snapshots from the semspect status endpoint. -- [**get_status**](#cmem_client.repositories.graph_insights.GraphInsightsRepository.get_status) – Fetch current status of a single snapshot. -- [**is_available**](#cmem_client.repositories.graph_insights.GraphInsightsRepository.is_available) – Return True if the semspect extension is active and the user is allowed. -- [**items**](#cmem_client.repositories.graph_insights.GraphInsightsRepository.items) – Get the items of the repository -- [**keys**](#cmem_client.repositories.graph_insights.GraphInsightsRepository.keys) – Get the keys of the repository -- [**update_item**](#cmem_client.repositories.graph_insights.GraphInsightsRepository.update_item) – Update an existing item in the repository. -- [**values**](#cmem_client.repositories.graph_insights.GraphInsightsRepository.values) – Get the values of the repository -- [**wait_for_completion**](#cmem_client.repositories.graph_insights.GraphInsightsRepository.wait_for_completion) – Poll until snapshot status is no longer ONGOING. +- [**create**](#cmem_client.repositories.graph_snapshots.GraphSnapshotsRepository.create) – Create or update a snapshot for the given graph IRI. +- [**delete_all**](#cmem_client.repositories.graph_snapshots.GraphSnapshotsRepository.delete_all) – Delete all snapshots via the bulk DELETE endpoint. +- [**delete_item**](#cmem_client.repositories.graph_snapshots.GraphSnapshotsRepository.delete_item) – Delete an item from the repository +- [**fetch_data**](#cmem_client.repositories.graph_snapshots.GraphSnapshotsRepository.fetch_data) – Fetch all snapshots from the semspect status endpoint. +- [**get_status**](#cmem_client.repositories.graph_snapshots.GraphSnapshotsRepository.get_status) – Fetch current status of a single snapshot. +- [**is_available**](#cmem_client.repositories.graph_snapshots.GraphSnapshotsRepository.is_available) – Deprecated: use `client.graphinsights.is_available()` instead. +- [**items**](#cmem_client.repositories.graph_snapshots.GraphSnapshotsRepository.items) – Get the items of the repository +- [**keys**](#cmem_client.repositories.graph_snapshots.GraphSnapshotsRepository.keys) – Get the keys of the repository +- [**update_item**](#cmem_client.repositories.graph_snapshots.GraphSnapshotsRepository.update_item) – Update an existing item in the repository. +- [**values**](#cmem_client.repositories.graph_snapshots.GraphSnapshotsRepository.values) – Get the values of the repository +- [**wait_for_completion**](#cmem_client.repositories.graph_snapshots.GraphSnapshotsRepository.wait_for_completion) – Poll until snapshot status is no longer ONGOING. **Attributes:** -- [**logger**](#cmem_client.repositories.graph_insights.GraphInsightsRepository.logger) (Logger) – Gets the client logger +- [**logger**](#cmem_client.repositories.graph_snapshots.GraphSnapshotsRepository.logger) (Logger) – Gets the client logger -### `create` {#cmem_client.repositories.graph_insights.GraphInsightsRepository.create} +### `create` {#cmem_client.repositories.graph_snapshots.GraphSnapshotsRepository.create} ```python create(iri) @@ -110,7 +111,7 @@ Create or update a snapshot for the given graph IRI. - str – The snapshot ID returned by the server. -### `delete_all` {#cmem_client.repositories.graph_insights.GraphInsightsRepository.delete_all} +### `delete_all` {#cmem_client.repositories.graph_snapshots.GraphSnapshotsRepository.delete_all} ```python delete_all() @@ -118,7 +119,7 @@ delete_all() Delete all snapshots via the bulk DELETE endpoint. -### `delete_item` {#cmem_client.repositories.graph_insights.GraphInsightsRepository.delete_item} +### `delete_item` {#cmem_client.repositories.graph_snapshots.GraphSnapshotsRepository.delete_item} ```python delete_item(key, skip_if_missing=False, configuration=None) @@ -137,7 +138,7 @@ Delete an item from the repository - [RepositoryModificationError](../../exceptions/index.md#cmem_client.exceptions.RepositoryModificationError) – if an error occurs while creating the item - HTTPError – for any other http error -### `fetch_data` {#cmem_client.repositories.graph_insights.GraphInsightsRepository.fetch_data} +### `fetch_data` {#cmem_client.repositories.graph_snapshots.GraphSnapshotsRepository.fetch_data} ```python fetch_data() @@ -145,7 +146,7 @@ fetch_data() Fetch all snapshots from the semspect status endpoint. -### `get_status` {#cmem_client.repositories.graph_insights.GraphInsightsRepository.get_status} +### `get_status` {#cmem_client.repositories.graph_snapshots.GraphSnapshotsRepository.get_status} ```python get_status(snapshot_id) @@ -161,15 +162,21 @@ Fetch current status of a single snapshot. - [GraphInsightSnapshot](../../models/graph_insight/index.md#cmem_client.models.graph_insight.GraphInsightSnapshot) – A GraphInsightSnapshot with current status fields. -### `is_available` {#cmem_client.repositories.graph_insights.GraphInsightsRepository.is_available} +### `is_available` {#cmem_client.repositories.graph_snapshots.GraphSnapshotsRepository.is_available} ```python is_available() ``` -Return True if the semspect extension is active and the user is allowed. +Deprecated: use `client.graphinsights.is_available()` instead. -### `items` {#cmem_client.repositories.graph_insights.GraphInsightsRepository.items} +Returns the same answer, and raises a `DeprecationWarning`. Removed in cmem-client 2.0.0. + +**Returns:** + +- bool – True if the Graph Insights extension is active and the user is allowed to use it. + +### `items` {#cmem_client.repositories.graph_snapshots.GraphSnapshotsRepository.items} ```python items() @@ -177,7 +184,7 @@ items() Get the items of the repository -### `keys` {#cmem_client.repositories.graph_insights.GraphInsightsRepository.keys} +### `keys` {#cmem_client.repositories.graph_snapshots.GraphSnapshotsRepository.keys} ```python keys() @@ -185,7 +192,7 @@ keys() Get the keys of the repository -### `logger` {#cmem_client.repositories.graph_insights.GraphInsightsRepository.logger} +### `logger` {#cmem_client.repositories.graph_snapshots.GraphSnapshotsRepository.logger} ```python logger: logging.Logger @@ -193,7 +200,7 @@ logger: logging.Logger Gets the client logger -### `update_item` {#cmem_client.repositories.graph_insights.GraphInsightsRepository.update_item} +### `update_item` {#cmem_client.repositories.graph_snapshots.GraphSnapshotsRepository.update_item} ```python update_item(item, configuration=None) @@ -211,7 +218,7 @@ Update an existing item in the repository. - [RepositoryModificationError](../../exceptions/index.md#cmem_client.exceptions.RepositoryModificationError) – If the item does not exist or an error occurs. - HTTPError – For any other HTTP error. -### `values` {#cmem_client.repositories.graph_insights.GraphInsightsRepository.values} +### `values` {#cmem_client.repositories.graph_snapshots.GraphSnapshotsRepository.values} ```python values() @@ -219,7 +226,7 @@ values() Get the values of the repository -### `wait_for_completion` {#cmem_client.repositories.graph_insights.GraphInsightsRepository.wait_for_completion} +### `wait_for_completion` {#cmem_client.repositories.graph_snapshots.GraphSnapshotsRepository.wait_for_completion} ```python wait_for_completion(snapshot_id, timeout=120.0, poll_interval=2.0) diff --git a/docs/develop/cmem-client-api/api-reference/repositories/graphs/index.md b/docs/develop/cmem-client-api/api-reference/repositories/graphs/index.md index 730f9f903..f8c7c1d27 100644 --- a/docs/develop/cmem-client-api/api-reference/repositories/graphs/index.md +++ b/docs/develop/cmem-client-api/api-reference/repositories/graphs/index.md @@ -58,7 +58,6 @@ Detect the serialization of a file before importing it: - [**GraphDeleteConfig**](#cmem_client.repositories.graphs.GraphDeleteConfig) – Graph Delete Configuration. - [**GraphExportConfig**](#cmem_client.repositories.graphs.GraphExportConfig) – Graph Export Configuration. -- [**GraphFileSerialization**](#cmem_client.repositories.graphs.GraphFileSerialization) – Supported graph format description - [**GraphImportConfig**](#cmem_client.repositories.graphs.GraphImportConfig) – Graph Import Configuration. - [**GraphsRepository**](#cmem_client.repositories.graphs.GraphsRepository) – Repository for graphs. @@ -99,7 +98,7 @@ Graph Export Configuration. **Attributes:** -- [**serialization**](#cmem_client.repositories.graphs.GraphExportConfig.serialization) ([GraphFileSerialization](#cmem_client.repositories.graphs.GraphFileSerialization) | None) – RDF serialization to request for the export. If None, the server default +- [**serialization**](#cmem_client.repositories.graphs.GraphExportConfig.serialization) ([GraphFileSerialization](../../models/graph/index.md#cmem_client.models.graph.GraphFileSerialization) | None) – RDF serialization to request for the export. If None, the server default is used. Export fails if the given format does not support export. - [**resolve_owl_imports**](#cmem_client.repositories.graphs.GraphExportConfig.resolve_owl_imports) (bool) – If True, resolve ``owl:imports`` and include the imported graphs in the export. Sent as ``owlImportsResolution``. @@ -122,67 +121,6 @@ resolve_owl_imports: bool = False serialization: GraphFileSerialization | None = None ``` -## `GraphFileSerialization` {#cmem_client.repositories.graphs.GraphFileSerialization} - -Bases: [Model](../../models/base/index.md#cmem_client.models.base.Model) - -Supported graph format description - -**Attributes:** - -- [**mime_type**](#cmem_client.repositories.graphs.GraphFileSerialization.mime_type) (str) – MIME type of the serialization, sent as ``Content-Type`` on import and as -``Accept`` on export. -- [**file_extensions**](#cmem_client.repositories.graphs.GraphFileSerialization.file_extensions) (list[str]) – File extensions mapped to this serialization, used by -``guess_file_type()`` to detect the format of a path. -- [**encoding**](#cmem_client.repositories.graphs.GraphFileSerialization.encoding) (str | None) – Content encoding of the file, sent as ``Content-Encoding`` on import when set. -- [**known_not_supporters**](#cmem_client.repositories.graphs.GraphFileSerialization.known_not_supporters) (list[str]) – Store types known not to support this serialization, matched against -the type reported by the graph store. The client does not enforce this; the test suite -uses it to skip combinations a store cannot handle. -- [**export_supported**](#cmem_client.repositories.graphs.GraphFileSerialization.export_supported) (bool) – Whether graphs can be exported in this serialization. -- [**import_supported**](#cmem_client.repositories.graphs.GraphFileSerialization.import_supported) (bool) – Whether graphs can be imported from this serialization. - -### `encoding` {#cmem_client.repositories.graphs.GraphFileSerialization.encoding} - -```python -encoding: str | None = None -``` - -### `export_supported` {#cmem_client.repositories.graphs.GraphFileSerialization.export_supported} - -```python -export_supported: bool = True -``` - -### `file_extensions` {#cmem_client.repositories.graphs.GraphFileSerialization.file_extensions} - -```python -file_extensions: list[str] -``` - -### `import_supported` {#cmem_client.repositories.graphs.GraphFileSerialization.import_supported} - -```python -import_supported: bool = True -``` - -### `known_not_supporters` {#cmem_client.repositories.graphs.GraphFileSerialization.known_not_supporters} - -```python -known_not_supporters: list[str] = Field(default_factory=list) -``` - -### `mime_type` {#cmem_client.repositories.graphs.GraphFileSerialization.mime_type} - -```python -mime_type: str -``` - -### `model_config` {#cmem_client.repositories.graphs.GraphFileSerialization.model_config} - -```python -model_config = ConfigDict(extra='allow', populate_by_name=True) -``` - ## `GraphImportConfig` {#cmem_client.repositories.graphs.GraphImportConfig} Bases: [ImportConfig](../../repositories/protocols/import_item/index.md#cmem_client.repositories.protocols.import_item.ImportConfig) @@ -192,7 +130,7 @@ Graph Import Configuration. **Attributes:** - [**register_as_vocabulary**](#cmem_client.repositories.graphs.GraphImportConfig.register_as_vocabulary) (bool) – If True, register the imported graph as a vocabulary. -- [**serialization**](#cmem_client.repositories.graphs.GraphImportConfig.serialization) ([GraphFileSerialization](#cmem_client.repositories.graphs.GraphFileSerialization) | None) – RDF serialization of the imported file. If None, it is guessed from the +- [**serialization**](#cmem_client.repositories.graphs.GraphImportConfig.serialization) ([GraphFileSerialization](../../models/graph/index.md#cmem_client.models.graph.GraphFileSerialization) | None) – RDF serialization of the imported file. If None, it is guessed from the file extension via ``guess_file_type()``. - [**namespace_prefix**](#cmem_client.repositories.graphs.GraphImportConfig.namespace_prefix) (str | None) – Vocabulary namespace prefix, used as a fallback when the file carries no vann metadata. Requires ``register_as_vocabulary=True`` and must be set together @@ -248,7 +186,7 @@ Supports both regular graphs and vocabularies through the register_as_vocabulary **Attributes:** -- [**formats**](#cmem_client.repositories.graphs.GraphsRepository.formats) (dict[str, [GraphFileSerialization](#cmem_client.repositories.graphs.GraphFileSerialization)]) – Registry of the supported RDF serializations, keyed by format name such as +- [**formats**](#cmem_client.repositories.graphs.GraphsRepository.formats) (dict[str, [GraphFileSerialization](../../models/graph/index.md#cmem_client.models.graph.GraphFileSerialization)]) – Registry of the supported RDF serializations, keyed by format name such as ``turtle`` or ``json-ld``. Read by ``guess_file_type()`` and available to callers which need to pick a serialization explicitly. @@ -332,7 +270,7 @@ Export an item from the repository to a file path. **Raises:** - [RepositoryItemNotFoundError](../../exceptions/index.md#cmem_client.exceptions.RepositoryItemNotFoundError) – If the specified item key is not found. -- [RepositoryReadError](../../exceptions/index.md#cmem_client.exceptions.RepositoryReadError) – If there's an error during export or path mismatch. +- [RepositoryReadError](../../exceptions/index.md#cmem_client.exceptions.RepositoryReadError) – If there is an error during export or path mismatch. ### `export_to_zip` {#cmem_client.repositories.graphs.GraphsRepository.export_to_zip} @@ -372,7 +310,7 @@ Use this method to fetch data when your result set is an array of objects. ### `formats` {#cmem_client.repositories.graphs.GraphsRepository.formats} ```python -formats: dict[str, GraphFileSerialization] = {'turtle': GraphFileSerialization(mime_type='text/turtle', file_extensions=['ttl']), 'rdf/xml': GraphFileSerialization(mime_type='application/rdf+xml', file_extensions=['rdf', 'xml']), 'json-ld': GraphFileSerialization(mime_type='application/ld+json', file_extensions=['jsonld'], known_not_supporters=['TENTRIS'], export_supported=False), 'n-triples': GraphFileSerialization(mime_type='application/n-triples', file_extensions=['nt']), 'pretty-turtle': GraphFileSerialization(mime_type='text/turtle+pretty', file_extensions=['ttl'], import_supported=False)} +formats: dict[str, GraphFileSerialization] = {'turtle': _GraphFileSerialization(mime_type='text/turtle', file_extensions=['ttl']), 'rdf/xml': _GraphFileSerialization(mime_type='application/rdf+xml', file_extensions=['rdf', 'xml']), 'json-ld': _GraphFileSerialization(mime_type='application/ld+json', file_extensions=['jsonld'], known_not_supporters=['TENTRIS'], export_supported=False), 'n-triples': _GraphFileSerialization(mime_type='application/n-triples', file_extensions=['nt']), 'pretty-turtle': _GraphFileSerialization(mime_type='text/turtle+pretty', file_extensions=['ttl'], import_supported=False)} ``` ### `guess_file_type` {#cmem_client.repositories.graphs.GraphsRepository.guess_file_type} @@ -393,7 +331,7 @@ Only considers formats where import_supported is True. **Returns:** -- **GraphFileSerialization** ([GraphFileSerialization](#cmem_client.repositories.graphs.GraphFileSerialization)) – The detected serialization format with +- **GraphFileSerialization** ([GraphFileSerialization](../../models/graph/index.md#cmem_client.models.graph.GraphFileSerialization)) – The detected serialization format with MIME type, file extensions, and optional encoding information. **Raises:** diff --git a/docs/develop/cmem-client-api/api-reference/repositories/marketplace_packages/index.md b/docs/develop/cmem-client-api/api-reference/repositories/marketplace_packages/index.md index 6e109f47a..de82ea29a 100644 --- a/docs/develop/cmem-client-api/api-reference/repositories/marketplace_packages/index.md +++ b/docs/develop/cmem-client-api/api-reference/repositories/marketplace_packages/index.md @@ -59,12 +59,13 @@ Install a package archive and export an installed package: **Functions:** +- [**get_file_resource_key**](#cmem_client.repositories.marketplace_packages.get_file_resource_key) – Get the composite key a packaged image or text file is stored under in Corporate Memory. - [**get_installation_metadata_query**](#cmem_client.repositories.marketplace_packages.get_installation_metadata_query) – Get the query for the installation metadata of the package. **Attributes:** - [**LOCK_FILE_RESOURCE**](#cmem_client.repositories.marketplace_packages.LOCK_FILE_RESOURCE) – -- [**MAX_DEPENDENCY_DEPTH**](#cmem_client.repositories.marketplace_packages.MAX_DEPENDENCY_DEPTH) – +- [**MAX_DEPENDENCY_DEPTH**](#cmem_client.repositories.marketplace_packages.MAX_DEPENDENCY_DEPTH) – Hard ceiling on marketplace-package dependency recursion depth. ## `LOCK_FILE_RESOURCE` {#cmem_client.repositories.marketplace_packages.LOCK_FILE_RESOURCE} @@ -75,9 +76,18 @@ LOCK_FILE_RESOURCE = f'{MARKETPLACE_PROJECT_ID}:mp-lock.json' ## `MAX_DEPENDENCY_DEPTH` {#cmem_client.repositories.marketplace_packages.MAX_DEPENDENCY_DEPTH} ```python -MAX_DEPENDENCY_DEPTH = 5 +MAX_DEPENDENCY_DEPTH = 20 ``` +Hard ceiling on marketplace-package dependency recursion depth. + +Not a cycle guard -- a real cycle is resolved for free by ``dependency_chain`` on +``MarketplacePackagesImportConfig``/``MarketplacePackagesDeleteConfig`` (see there), which +stops re-descending into a package already in progress higher up the same chain, whatever the +depth. This exists for a pathologically deep-but-acyclic chain, or a misbehaving marketplace. +The number is generous headroom above any realistic dependency chain, not a meaningful +business limit. + ## `MarketplacePackagesDeleteConfig` {#cmem_client.repositories.marketplace_packages.MarketplacePackagesDeleteConfig} Bases: [DeleteConfig](../../repositories/protocols/delete_item/index.md#cmem_client.repositories.protocols.delete_item.DeleteConfig) @@ -92,10 +102,25 @@ instead of raising an error. instead of raising an error. - [**skip_missing_projects**](#cmem_client.repositories.marketplace_packages.MarketplacePackagesDeleteConfig.skip_missing_projects) (bool) – If True, projects of the package which do not exist are skipped instead of raising an error. +- [**skip_missing_views**](#cmem_client.repositories.marketplace_packages.MarketplacePackagesDeleteConfig.skip_missing_views) (bool) – If True, application views of the package which do not exist (for +example because somebody deleted them by hand after the installation) are skipped +instead of raising an error. - [**ignore_lock**](#cmem_client.repositories.marketplace_packages.MarketplacePackagesDeleteConfig.ignore_lock) (bool) – If set to True, ignore the lock mechanism. - [**ignore_dependencies**](#cmem_client.repositories.marketplace_packages.MarketplacePackagesDeleteConfig.ignore_dependencies) (bool) – If True, dependencies of the package are not deleted. - [**dependency_level**](#cmem_client.repositories.marketplace_packages.MarketplacePackagesDeleteConfig.dependency_level) (int) – Current recursion depth for dependency resolution. Used internally to -identify the top-level call. Should not be set manually. +identify the top-level call and to enforce MAX_DEPENDENCY_DEPTH. Should not be set +manually. +- [**dependency_chain**](#cmem_client.repositories.marketplace_packages.MarketplacePackagesDeleteConfig.dependency_chain) (tuple[PackageIdentifier, ...]) – package_ids of the packages currently being deleted, from the +top-level call down to (but not including) the package about to be deleted. Used +internally to skip re-descending into a package a dependency cycle leads back +to -- it is already being deleted higher up this chain. Should not be set +manually. + +### `dependency_chain` {#cmem_client.repositories.marketplace_packages.MarketplacePackagesDeleteConfig.dependency_chain} + +```python +dependency_chain: tuple[PackageIdentifier, ...] = () +``` ### `dependency_level` {#cmem_client.repositories.marketplace_packages.MarketplacePackagesDeleteConfig.dependency_level} @@ -139,6 +164,12 @@ skip_missing_graphs: bool = True skip_missing_projects: bool = True ``` +### `skip_missing_views` {#cmem_client.repositories.marketplace_packages.MarketplacePackagesDeleteConfig.skip_missing_views} + +```python +skip_missing_views: bool = True +``` + ## `MarketplacePackagesExportConfig` {#cmem_client.repositories.marketplace_packages.MarketplacePackagesExportConfig} Bases: [ExportConfig](../../repositories/protocols/export_item/index.md#cmem_client.repositories.protocols.export_item.ExportConfig) @@ -181,10 +212,44 @@ Configuration for marketplace package import operations. If False, loads packages from local filesystem. - [**package_version**](#cmem_client.repositories.marketplace_packages.MarketplacePackagesImportConfig.package_version) (PackageVersionIdentifier | None) – Specific version to install. If None, installs the latest version. - [**dependency_level**](#cmem_client.repositories.marketplace_packages.MarketplacePackagesImportConfig.dependency_level) (int) – Current recursion depth for dependency resolution. Used internally -to prevent infinite recursion. Should not be set manually. +to enforce MAX_DEPENDENCY_DEPTH. Should not be set manually. +- [**dependency_chain**](#cmem_client.repositories.marketplace_packages.MarketplacePackagesImportConfig.dependency_chain) (tuple[PackageIdentifier, ...]) – package_ids of the packages currently being installed, from the +top-level call down to (but not including) the package about to be installed. +Used internally to skip re-descending into a package a dependency cycle leads + +back to -- it is already being installed higher up this chain. Should not be +set manually. + - [**use_cache**](#cmem_client.repositories.marketplace_packages.MarketplacePackagesImportConfig.use_cache) (bool) – Weather to use the cache directory to look packages up which have already been downloaded. To prevent the cache entirely, set this up in the marketplace component. - [**ignore_lock**](#cmem_client.repositories.marketplace_packages.MarketplacePackagesImportConfig.ignore_lock) (bool) – If set to True, ignore the lock mechanism. +- [**force_dependencies**](#cmem_client.repositories.marketplace_packages.MarketplacePackagesImportConfig.force_dependencies) (bool) – If True, an installed dependency whose version violates the +constraint a package declares on it is upgraded or downgraded to a version which +satisfies the constraint, and the installation proceeds although a version it puts +in place (the package itself, or a dependency) violates the constraint of an +installed package, which is then listed in ``broken_dependants``. If False, such +an installation fails before anything is installed. +- [**resolution**](#cmem_client.repositories.marketplace_packages.MarketplacePackagesImportConfig.resolution) (DependencyResolution | None) – Resolved dependency versions of the whole installation tree. Used +internally to hand the resolution down to dependency installations. Should not +be set manually. +- [**source**](#cmem_client.repositories.marketplace_packages.MarketplacePackagesImportConfig.source) (PackageSource | None) – Where versions, dependency lists and archives are read from. If None, the +marketplace server of the client (``client.marketplace``) is used. +- [**broken_dependants**](#cmem_client.repositories.marketplace_packages.MarketplacePackagesImportConfig.broken_dependants) (list[BrokenDependant]) – Output of the installation: the installed packages whose version +constraint it broke because the dependencies were forced. Cleared at the start of +every top-level installation, so it always describes the last one made with this +configuration. Empty after an installation which broke nothing. + +### `broken_dependants` {#cmem_client.repositories.marketplace_packages.MarketplacePackagesImportConfig.broken_dependants} + +```python +broken_dependants: list[BrokenDependant] = Field(default_factory=list) +``` + +### `dependency_chain` {#cmem_client.repositories.marketplace_packages.MarketplacePackagesImportConfig.dependency_chain} + +```python +dependency_chain: tuple[PackageIdentifier, ...] = () +``` ### `dependency_level` {#cmem_client.repositories.marketplace_packages.MarketplacePackagesImportConfig.dependency_level} @@ -192,6 +257,12 @@ To prevent the cache entirely, set this up in the marketplace component. dependency_level: int = 0 ``` +### `force_dependencies` {#cmem_client.repositories.marketplace_packages.MarketplacePackagesImportConfig.force_dependencies} + +```python +force_dependencies: bool = False +``` + ### `ignore_dependencies` {#cmem_client.repositories.marketplace_packages.MarketplacePackagesImportConfig.ignore_dependencies} ```python @@ -213,7 +284,7 @@ install_from_marketplace: bool = True ### `model_config` {#cmem_client.repositories.marketplace_packages.MarketplacePackagesImportConfig.model_config} ```python -model_config = ConfigDict(extra='allow', populate_by_name=True) +model_config = ConfigDict(arbitrary_types_allowed=True) ``` ### `package_version` {#cmem_client.repositories.marketplace_packages.MarketplacePackagesImportConfig.package_version} @@ -222,6 +293,18 @@ model_config = ConfigDict(extra='allow', populate_by_name=True) package_version: PackageVersionIdentifier | None = None ``` +### `resolution` {#cmem_client.repositories.marketplace_packages.MarketplacePackagesImportConfig.resolution} + +```python +resolution: DependencyResolution | None = None +``` + +### `source` {#cmem_client.repositories.marketplace_packages.MarketplacePackagesImportConfig.source} + +```python +source: PackageSource | None = None +``` + ### `use_archive_handler` {#cmem_client.repositories.marketplace_packages.MarketplacePackagesImportConfig.use_archive_handler} ```python @@ -242,19 +325,55 @@ Repository for marketplace package operations. **Functions:** +- [**acquire_lock**](#cmem_client.repositories.marketplace_packages.MarketplacePackagesRepository.acquire_lock) – Acquire the shared marketplace-packages install/uninstall lock for a whole operation. - [**delete_all**](#cmem_client.repositories.marketplace_packages.MarketplacePackagesRepository.delete_all) – Delete all items from the repository - [**delete_item**](#cmem_client.repositories.marketplace_packages.MarketplacePackagesRepository.delete_item) – Delete an item from the repository +- [**dependency_tree**](#cmem_client.repositories.marketplace_packages.MarketplacePackagesRepository.dependency_tree) – Report the dependency tree of the installed packages. - [**export_item**](#cmem_client.repositories.marketplace_packages.MarketplacePackagesRepository.export_item) – Export an item from the repository to a file path. - [**fetch_data**](#cmem_client.repositories.marketplace_packages.MarketplacePackagesRepository.fetch_data) – Fetch installed packages from the package data graph via SPARQL query. +- [**get_installed_file**](#cmem_client.repositories.marketplace_packages.MarketplacePackagesRepository.get_installed_file) – Read a packaged image or text file of an installed package back from Corporate Memory. - [**import_item**](#cmem_client.repositories.marketplace_packages.MarketplacePackagesRepository.import_item) – Import an exported file to the repository - [**items**](#cmem_client.repositories.marketplace_packages.MarketplacePackagesRepository.items) – Get the items of the repository - [**keys**](#cmem_client.repositories.marketplace_packages.MarketplacePackagesRepository.keys) – Get the keys of the repository +- [**release_lock**](#cmem_client.repositories.marketplace_packages.MarketplacePackagesRepository.release_lock) – Release the shared marketplace-packages install/uninstall lock acquired by `acquire_lock`. - [**values**](#cmem_client.repositories.marketplace_packages.MarketplacePackagesRepository.values) – Get the values of the repository **Attributes:** - [**logger**](#cmem_client.repositories.marketplace_packages.MarketplacePackagesRepository.logger) (Logger) – Gets the client logger +### `acquire_lock` {#cmem_client.repositories.marketplace_packages.MarketplacePackagesRepository.acquire_lock} + +```python +acquire_lock(package_id, activity='Install') +``` + +Acquire the shared marketplace-packages install/uninstall lock for a whole operation. + +Import/delete each threads a fragile ``ignore_lock``/``dependency_level`` pair through +every recursive call to know which one is responsible for acquiring and releasing the +shared lock -- correct for a plain recursive walk, but it breaks down when a step in +that walk is skipped for an unrelated reason (e.g. a dependency turns out to already be +installed): nothing acquires the lock in that call's place, and a caller that assumed it +had can go on to (wrongly) treat a lock still held from a genuinely separate, earlier, +possibly-orphaned operation as its own. + +Call this once, explicitly, before a multi-step operation that composes several +import/delete calls into one logical whole (see e.g. +``eccenca_marketplace.services.install_local_package``), then pass ``ignore_lock=True`` +to every one of those calls -- no per-call bookkeeping needed. Pair with `release_lock` +in a ``finally``, regardless of whether every step actually ran. + +**Parameters:** + +- **package_id** (str) – The package this operation is ultimately for -- recorded in the lock, +and named in the error message if another process already holds it. +- **activity** (Literal['Install', 'Uninstall']) – What the lock records itself as doing. + +**Raises:** + +- [MarketplacePackagesImportError](../../exceptions/index.md#cmem_client.exceptions.MarketplacePackagesImportError) – If another process already holds the lock. + ### `delete_all` {#cmem_client.repositories.marketplace_packages.MarketplacePackagesRepository.delete_all} ```python @@ -282,6 +401,37 @@ Delete an item from the repository - [RepositoryModificationError](../../exceptions/index.md#cmem_client.exceptions.RepositoryModificationError) – if an error occurs while creating the item - HTTPError – for any other http error +### `dependency_tree` {#cmem_client.repositories.marketplace_packages.MarketplacePackagesRepository.dependency_tree} + +```python +dependency_tree(package_id=None) +``` + +Report the dependency tree of the installed packages. + +Every installed package is listed with the dependencies its manifest declares - +marketplace packages and Python packages alike - each with the declared constraint, +the installed version and whether it is satisfied, violated or missing, and with the +installed packages which depend on it. The violations of the tree collect every +violated and missing dependency, so an empty list means the installed tree is +consistent. A dependency which is missing because its dependant was installed while +ignoring the dependencies, or because it was uninstalled, is reported as missing. +The roots of the tree are the installed packages nothing else requires, and the +nested tree expands them, every package once, for a caller which renders it. + +**Parameters:** + +- **package_id** (str | None) – Report only the part of the tree this installed package pulls in, +rooted at it (see ``installed_subtree()``). + +**Returns:** + +- DependencyTree – The tree, read from Corporate Memory right now. + +**Raises:** + +- [MarketplacePackagesNotFoundError](../../exceptions/index.md#cmem_client.exceptions.MarketplacePackagesNotFoundError) – If ``package_id`` is not installed. + ### `export_item` {#cmem_client.repositories.marketplace_packages.MarketplacePackagesRepository.export_item} ```python @@ -304,7 +454,7 @@ Export an item from the repository to a file path. **Raises:** - [RepositoryItemNotFoundError](../../exceptions/index.md#cmem_client.exceptions.RepositoryItemNotFoundError) – If the specified item key is not found. -- [RepositoryReadError](../../exceptions/index.md#cmem_client.exceptions.RepositoryReadError) – If there's an error during export or path mismatch. +- [RepositoryReadError](../../exceptions/index.md#cmem_client.exceptions.RepositoryReadError) – If there is an error during export or path mismatch. ### `fetch_data` {#cmem_client.repositories.marketplace_packages.MarketplacePackagesRepository.fetch_data} @@ -316,6 +466,35 @@ Fetch installed packages from the package data graph via SPARQL query. Queries the package data graph for all installed packages and their metadata. +### `get_installed_file` {#cmem_client.repositories.marketplace_packages.MarketplacePackagesRepository.get_installed_file} + +```python +get_installed_file(package_id, file_path) +``` + +Read a packaged image or text file of an installed package back from Corporate Memory. + +An installation keeps the icon, the marketplace images, the readme, the license and +the changelog of a package as file resources, whether the package came from a +marketplace or from a file, so they can be shown after the archive is gone. Only a +path the installed manifest declares as an image or text file can be read. + +**Parameters:** + +- **package_id** (str) – The ID of the installed package. +- **file_path** (str) – The path of the file as the installed manifest declares it. + +**Returns:** + +- bytes – The raw content of the file. + +**Raises:** + +- [MarketplacePackagesNotFoundError](../../exceptions/index.md#cmem_client.exceptions.MarketplacePackagesNotFoundError) – If the package is not installed. +- [MarketplacePackagesFileNotFoundError](../../exceptions/index.md#cmem_client.exceptions.MarketplacePackagesFileNotFoundError) – If the installed manifest declares no image or +text file with this path. +- FilesNotFoundError – If the file resource is missing in Corporate Memory. + ### `import_item` {#cmem_client.repositories.marketplace_packages.MarketplacePackagesRepository.import_item} ```python @@ -362,6 +541,17 @@ logger: logging.Logger Gets the client logger +### `release_lock` {#cmem_client.repositories.marketplace_packages.MarketplacePackagesRepository.release_lock} + +```python +release_lock() +``` + +Release the shared marketplace-packages install/uninstall lock acquired by `acquire_lock`. + +Safe to call even if the assets project or the lock file no longer exist (e.g. the +last package was also removed as part of the same operation). + ### `values` {#cmem_client.repositories.marketplace_packages.MarketplacePackagesRepository.values} ```python @@ -370,6 +560,14 @@ values() Get the values of the repository +## `get_file_resource_key` {#cmem_client.repositories.marketplace_packages.get_file_resource_key} + +```python +get_file_resource_key(package_id, file_path) +``` + +Get the composite key a packaged image or text file is stored under in Corporate Memory. + ## `get_installation_metadata_query` {#cmem_client.repositories.marketplace_packages.get_installation_metadata_query} ```python diff --git a/docs/develop/cmem-client-api/api-reference/repositories/projects/index.md b/docs/develop/cmem-client-api/api-reference/repositories/projects/index.md index 65a9ab1d7..0a067f5f7 100644 --- a/docs/develop/cmem-client-api/api-reference/repositories/projects/index.md +++ b/docs/develop/cmem-client-api/api-reference/repositories/projects/index.md @@ -14,7 +14,8 @@ tags: Repository for managing DataIntegration projects. Provides ProjectsRepository for creating, deleting, importing and exporting build -projects, and for reloading a project and reading its failed task report. +projects, and for reloading a project, reading its failed task report and its prefix +map. **Examples:** @@ -46,53 +47,20 @@ Find out which tasks of a project failed to load: ... print(failed) ``` +Get the prefix map of a project: + +```pycon +>>> client.projects.get_prefixes("my-project") +``` + **Classes:** -- [**ProjectImportStatus**](#cmem_client.repositories.projects.ProjectImportStatus) – Response of the project import status endpoint. - [**ProjectsCreateConfig**](#cmem_client.repositories.projects.ProjectsCreateConfig) – Project Create Configuration. - [**ProjectsDeleteConfig**](#cmem_client.repositories.projects.ProjectsDeleteConfig) – Project Delete Configuration. - [**ProjectsExportConfig**](#cmem_client.repositories.projects.ProjectsExportConfig) – Project Export Configuration. - [**ProjectsImportConfig**](#cmem_client.repositories.projects.ProjectsImportConfig) – Project Import Configuration. - [**ProjectsRepository**](#cmem_client.repositories.projects.ProjectsRepository) – Repository for Build (DataIntegration) projects. -## `ProjectImportStatus` {#cmem_client.repositories.projects.ProjectImportStatus} - -Bases: [Model](../../models/base/index.md#cmem_client.models.base.Model) - -Response of the project import status endpoint. - -**Attributes:** - -- [**project_id**](#cmem_client.repositories.projects.ProjectImportStatus.project_id) (str) – Identifier of the imported project, sent as ``projectId``. -- [**success**](#cmem_client.repositories.projects.ProjectImportStatus.success) (bool | None) – Whether the import finished successfully. None while the import is still -running, which is what ``import_item()`` polls on. -- [**failure_message**](#cmem_client.repositories.projects.ProjectImportStatus.failure_message) (str | None) – Reason the import failed, sent as ``failureMessage``. Only set when -the import failed. - -### `failure_message` {#cmem_client.repositories.projects.ProjectImportStatus.failure_message} - -```python -failure_message: str | None = Field(alias='failureMessage', default=None) -``` - -### `model_config` {#cmem_client.repositories.projects.ProjectImportStatus.model_config} - -```python -model_config = ConfigDict(extra='allow', populate_by_name=True) -``` - -### `project_id` {#cmem_client.repositories.projects.ProjectImportStatus.project_id} - -```python -project_id: str = Field(alias='projectId') -``` - -### `success` {#cmem_client.repositories.projects.ProjectImportStatus.success} - -```python -success: bool | None = None -``` - ## `ProjectsCreateConfig` {#cmem_client.repositories.projects.ProjectsCreateConfig} Bases: [CreateConfig](../../repositories/protocols/create_item/index.md#cmem_client.repositories.protocols.create_item.CreateConfig) @@ -207,6 +175,7 @@ the [Project model](../../models/project/index.md#cmem_client.models.project.Pro - [**export_item**](#cmem_client.repositories.projects.ProjectsRepository.export_item) – Export an item from the repository to a file path. - [**fetch_data**](#cmem_client.repositories.projects.ProjectsRepository.fetch_data) – Fetch simple list from a JSON endpoint via a type adapter - [**get_failed_tasks_report**](#cmem_client.repositories.projects.ProjectsRepository.get_failed_tasks_report) – Get all failed tasks from project from its ID +- [**get_prefixes**](#cmem_client.repositories.projects.ProjectsRepository.get_prefixes) – Get the prefix map (prefix to namespace URI) of a project. - [**import_item**](#cmem_client.repositories.projects.ProjectsRepository.import_item) – Import an exported file to the repository - [**items**](#cmem_client.repositories.projects.ProjectsRepository.items) – Get the items of the repository - [**keys**](#cmem_client.repositories.projects.ProjectsRepository.keys) – Get the keys of the repository @@ -286,7 +255,7 @@ Export an item from the repository to a file path. **Raises:** - [RepositoryItemNotFoundError](../../exceptions/index.md#cmem_client.exceptions.RepositoryItemNotFoundError) – If the specified item key is not found. -- [RepositoryReadError](../../exceptions/index.md#cmem_client.exceptions.RepositoryReadError) – If there's an error during export or path mismatch. +- [RepositoryReadError](../../exceptions/index.md#cmem_client.exceptions.RepositoryReadError) – If there is an error during export or path mismatch. ### `fetch_data` {#cmem_client.repositories.projects.ProjectsRepository.fetch_data} @@ -306,6 +275,14 @@ get_failed_tasks_report(project_id) Get all failed tasks from project from its ID +### `get_prefixes` {#cmem_client.repositories.projects.ProjectsRepository.get_prefixes} + +```python +get_prefixes(project_id) +``` + +Get the prefix map (prefix to namespace URI) of a project. + ### `import_item` {#cmem_client.repositories.projects.ProjectsRepository.import_item} ```python diff --git a/docs/develop/cmem-client-api/api-reference/repositories/protocols/export_item/index.md b/docs/develop/cmem-client-api/api-reference/repositories/protocols/export_item/index.md index 040d29b3c..17a701824 100644 --- a/docs/develop/cmem-client-api/api-reference/repositories/protocols/export_item/index.md +++ b/docs/develop/cmem-client-api/api-reference/repositories/protocols/export_item/index.md @@ -85,7 +85,7 @@ Export an item from the repository to a file path. **Raises:** - [RepositoryItemNotFoundError](../../../exceptions/index.md#cmem_client.exceptions.RepositoryItemNotFoundError) – If the specified item key is not found. -- [RepositoryReadError](../../../exceptions/index.md#cmem_client.exceptions.RepositoryReadError) – If there's an error during export or path mismatch. +- [RepositoryReadError](../../../exceptions/index.md#cmem_client.exceptions.RepositoryReadError) – If there is an error during export or path mismatch. ### `logger` {#cmem_client.repositories.protocols.export_item.ExportItemProtocol.logger} diff --git a/docs/develop/cmem-client-api/api-reference/repositories/queries/index.md b/docs/develop/cmem-client-api/api-reference/repositories/queries/index.md index fbf6db3d3..e6e071a87 100644 --- a/docs/develop/cmem-client-api/api-reference/repositories/queries/index.md +++ b/docs/develop/cmem-client-api/api-reference/repositories/queries/index.md @@ -364,7 +364,7 @@ Export an item from the repository to a file path. **Raises:** - [RepositoryItemNotFoundError](../../exceptions/index.md#cmem_client.exceptions.RepositoryItemNotFoundError) – If the specified item key is not found. -- [RepositoryReadError](../../exceptions/index.md#cmem_client.exceptions.RepositoryReadError) – If there's an error during export or path mismatch. +- [RepositoryReadError](../../exceptions/index.md#cmem_client.exceptions.RepositoryReadError) – If there is an error during export or path mismatch. ### `fetch_data` {#cmem_client.repositories.queries.QueriesRepository.fetch_data} diff --git a/docs/develop/cmem-client-api/api-reference/repositories/rule_blocks/index.md b/docs/develop/cmem-client-api/api-reference/repositories/rule_blocks/index.md new file mode 100644 index 000000000..1ccd9af6b --- /dev/null +++ b/docs/develop/cmem-client-api/api-reference/repositories/rule_blocks/index.md @@ -0,0 +1,269 @@ +--- +title: "cmem-client: repositories.rule_blocks module" +description: "Rule block repository." +tags: + - API + - Python + - cmem-client +--- + +# `cmem_client.repositories.rule_blocks` {#cmem_client.repositories.rule_blocks} + + + +Rule block repository. + +Rule blocks are listed through the generic task search endpoint. Because +``TaskSearchRepository.fetch_data`` already requests task parameters, the search +response carries the complete ``ruleBlockModel`` including the declared input +ports, so no additional request is needed to report them. + +**Classes:** + +- [**RuleBlockDeleteConfig**](#cmem_client.repositories.rule_blocks.RuleBlockDeleteConfig) – Rule block deletion configuration. +- [**RuleBlocksRepository**](#cmem_client.repositories.rule_blocks.RuleBlocksRepository) – Repository for rule blocks. + +## `RuleBlockDeleteConfig` {#cmem_client.repositories.rule_blocks.RuleBlockDeleteConfig} + +Bases: [DeleteConfig](../../repositories/protocols/delete_item/index.md#cmem_client.repositories.protocols.delete_item.DeleteConfig) + +Rule block deletion configuration. + +``remove_dependent_tasks`` maps to the required ``removeDependentTasks`` +query parameter of the task delete endpoint. When true, DataIntegration +deletes every task that directly *or indirectly* references the rule block. +When false, referencing tasks are left behind holding a dangling reference, +which DataIntegration reports nowhere - callers are expected to check for +dependents themselves before passing false. + +**Attributes:** + +- [**model_config**](#cmem_client.repositories.rule_blocks.RuleBlockDeleteConfig.model_config) – +- [**remove_dependent_tasks**](#cmem_client.repositories.rule_blocks.RuleBlockDeleteConfig.remove_dependent_tasks) (bool) – + +### `model_config` {#cmem_client.repositories.rule_blocks.RuleBlockDeleteConfig.model_config} + +```python +model_config = ConfigDict(extra='allow', populate_by_name=True) +``` + +### `remove_dependent_tasks` {#cmem_client.repositories.rule_blocks.RuleBlockDeleteConfig.remove_dependent_tasks} + +```python +remove_dependent_tasks: bool = False +``` + +## `RuleBlocksRepository` {#cmem_client.repositories.rule_blocks.RuleBlocksRepository} + +Bases: [TaskSearchRepository](../../repositories/base/task_search/index.md#cmem_client.repositories.base.task_search.TaskSearchRepository), [DeleteItemProtocol](../../repositories/protocols/delete_item/index.md#cmem_client.repositories.protocols.delete_item.DeleteItemProtocol) + +Repository for rule blocks. + +**Functions:** + +- [**create_item**](#cmem_client.repositories.rule_blocks.RuleBlocksRepository.create_item) – Create a new rule block in a project. +- [**delete_all**](#cmem_client.repositories.rule_blocks.RuleBlocksRepository.delete_all) – Delete all items from the repository +- [**delete_item**](#cmem_client.repositories.rule_blocks.RuleBlocksRepository.delete_item) – Delete an item from the repository +- [**evaluate**](#cmem_client.repositories.rule_blocks.RuleBlocksRepository.evaluate) – Evaluate a rule block against the input examples stored with it. +- [**fetch_data**](#cmem_client.repositories.rule_blocks.RuleBlocksRepository.fetch_data) – Fetch a list from the DI task search endpoint via a type adapter. +- [**from_task_response**](#cmem_client.repositories.rule_blocks.RuleBlocksRepository.from_task_response) – Reshape a task detail response into the canonical RuleBlock structure. +- [**get_item**](#cmem_client.repositories.rule_blocks.RuleBlocksRepository.get_item) – Get full rule block details including metadata timestamps. +- [**get_task**](#cmem_client.repositories.rule_blocks.RuleBlocksRepository.get_task) – Get full task details from the API. +- [**items**](#cmem_client.repositories.rule_blocks.RuleBlocksRepository.items) – Get the items of the repository +- [**keys**](#cmem_client.repositories.rule_blocks.RuleBlocksRepository.keys) – Get the keys of the repository +- [**update_item**](#cmem_client.repositories.rule_blocks.RuleBlocksRepository.update_item) – Replace an existing rule block with the given one. +- [**values**](#cmem_client.repositories.rule_blocks.RuleBlocksRepository.values) – Get the values of the repository + +**Attributes:** + +- [**logger**](#cmem_client.repositories.rule_blocks.RuleBlocksRepository.logger) (Logger) – Gets the client logger + +### `create_item` {#cmem_client.repositories.rule_blocks.RuleBlocksRepository.create_item} + +```python +create_item(item) +``` + +Create a new rule block in a project. + +**Parameters:** + +- **item** ([RuleBlock](../../models/rule_block/index.md#cmem_client.models.rule_block.RuleBlock)) – RuleBlock with ``project_id``, ``id``, ``metadata`` and +optionally ``data.rule_block_model``. + +**Returns:** + +- [RuleBlock](../../models/rule_block/index.md#cmem_client.models.rule_block.RuleBlock) – The created rule block, read back from the server. + +**Raises:** + +- HTTPStatusError – If the creation request fails. + +### `delete_all` {#cmem_client.repositories.rule_blocks.RuleBlocksRepository.delete_all} + +```python +delete_all() +``` + +Delete all items from the repository + +### `delete_item` {#cmem_client.repositories.rule_blocks.RuleBlocksRepository.delete_item} + +```python +delete_item(key, skip_if_missing=False, configuration=None) +``` + +Delete an item from the repository + +**Parameters:** + +- **key** (str) – The key of the item to delete +- **skip_if_missing** (bool) – If True, it is ignored if the deleted item even exists +- **configuration** (DeleteItemConfig) – Optional configuration for deletion + +**Raises:** + +- [RepositoryModificationError](../../exceptions/index.md#cmem_client.exceptions.RepositoryModificationError) – if an error occurs while creating the item +- HTTPError – for any other http error + +### `evaluate` {#cmem_client.repositories.rule_blocks.RuleBlocksRepository.evaluate} + +```python +evaluate(project_id, rule_block_id, model) +``` + +Evaluate a rule block against the input examples stored with it. + +Both the project and the rule block task must exist; the endpoint is +task-scoped and there is no project-free evaluation mode. + +**Parameters:** + +- **project_id** (str) – The project ID. +- **rule_block_id** (str) – The rule block ID. +- **model** ([RuleBlockModel](../../models/rule_block/index.md#cmem_client.models.rule_block.RuleBlockModel)) – The rule block model to evaluate. + +**Returns:** + +- list[Any] – One result entry per stored input example. + +**Raises:** + +- HTTPStatusError – If the project or rule block does not exist. + +### `fetch_data` {#cmem_client.repositories.rule_blocks.RuleBlocksRepository.fetch_data} + +```python +fetch_data() +``` + +Fetch a list from the DI task search endpoint via a type adapter. + +### `from_task_response` {#cmem_client.repositories.rule_blocks.RuleBlocksRepository.from_task_response} + +```python +from_task_response(data) +``` + +Reshape a task detail response into the canonical RuleBlock structure. + +The task detail endpoint nests the model under ``data.parameters``. It is +requested without labels, because with labels every parameter value is +wrapped into an additional ``value`` object. + +**Parameters:** + +- **data** (dict[str, Any]) – The JSON response of the task detail endpoint. + +**Returns:** + +- [RuleBlock](../../models/rule_block/index.md#cmem_client.models.rule_block.RuleBlock) – The rule block with the ``ruleBlockModel`` parameter moved under ``data``. + +### `get_item` {#cmem_client.repositories.rule_blocks.RuleBlocksRepository.get_item} + +```python +get_item(project_id, rule_block_id) +``` + +Get full rule block details including metadata timestamps. + +**Parameters:** + +- **project_id** (str) – The project ID. +- **rule_block_id** (str) – The rule block ID. + +**Returns:** + +- [RuleBlock](../../models/rule_block/index.md#cmem_client.models.rule_block.RuleBlock) – RuleBlock model with the full model and metadata. + +**Raises:** + +- HTTPStatusError – If the rule block is not found. + +### `get_task` {#cmem_client.repositories.rule_blocks.RuleBlocksRepository.get_task} + +```python +get_task(project_id, task_id, with_labels=True) +``` + +Get full task details from the API. + +**Parameters:** + +- **project_id** (str) – The project ID. +- **task_id** (str) – The task ID. +- **with_labels** (bool) – Whether to include labels in the response. + +**Returns:** + +- [TaskResponse](../../models/task/index.md#cmem_client.models.task.TaskResponse) – The full task details as a TaskResponse model. + +### `items` {#cmem_client.repositories.rule_blocks.RuleBlocksRepository.items} + +```python +items() +``` + +Get the items of the repository + +### `keys` {#cmem_client.repositories.rule_blocks.RuleBlocksRepository.keys} + +```python +keys() +``` + +Get the keys of the repository + +### `logger` {#cmem_client.repositories.rule_blocks.RuleBlocksRepository.logger} + +```python +logger: logging.Logger +``` + +Gets the client logger + +### `update_item` {#cmem_client.repositories.rule_blocks.RuleBlocksRepository.update_item} + +```python +update_item(item) +``` + +Replace an existing rule block with the given one. + +**Parameters:** + +- **item** ([RuleBlock](../../models/rule_block/index.md#cmem_client.models.rule_block.RuleBlock)) – RuleBlock with ``project_id``, ``id``, ``metadata`` and +``data.rule_block_model``. + +**Raises:** + +- HTTPStatusError – If the update request fails. + +### `values` {#cmem_client.repositories.rule_blocks.RuleBlocksRepository.values} + +```python +values() +``` + +Get the values of the repository + diff --git a/docs/develop/cmem-client-api/api-reference/repositories/variables/index.md b/docs/develop/cmem-client-api/api-reference/repositories/variables/index.md index 9e6d860ed..80edcbae5 100644 --- a/docs/develop/cmem-client-api/api-reference/repositories/variables/index.md +++ b/docs/develop/cmem-client-api/api-reference/repositories/variables/index.md @@ -13,9 +13,11 @@ tags: Repository for the variables of DataIntegration projects. -Provides VariablesRepository for creating, updating and deleting project variables. -Variables are keyed by the composite key ``project_id:variable_name``, so a single -repository spans the variables of all projects. +Provides VariablesRepository for creating, updating and deleting project variables and +the execution variables of tasks. Project variables are keyed by +``project_id:variable_name``, execution variables by +``project_id:task_id:variable_name``, so a single repository spans the variables of all +projects and their tasks. **Examples:** @@ -32,11 +34,27 @@ Create a variable in a project and read its value back: >>> client.variables.get_item("my-project", "greeting").value ``` -List every variable of the deployment and delete one: +Create an execution variable on a task and read its value back: + +```pycon +>>> client.variables.create_item( +... Variable( +... name="limit", project_id="my-project", task_id="my-workflow", value="10", scope="execution" +... ) +... ) +>>> client.variables.get_item("my-project", "limit", task_id="my-workflow").value +``` + +See what still uses a variable, list every variable of the deployment and +delete one: + +```pycon +>>> client.variables.get_dependents("my-project:greeting") +``` ```pycon >>> list(client.variables) ->>> client.variables.delete_item("my-project:greeting") +>>> client.variables.delete_item("my-project:my-workflow:limit") ``` **Classes:** @@ -44,7 +62,7 @@ List every variable of the deployment and delete one: - [**VariableCreateConfig**](#cmem_client.repositories.variables.VariableCreateConfig) – Variable creation configuration. - [**VariableDeleteConfig**](#cmem_client.repositories.variables.VariableDeleteConfig) – Variable deletion configuration. - [**VariableUpdateConfig**](#cmem_client.repositories.variables.VariableUpdateConfig) – Variable update configuration. -- [**VariablesRepository**](#cmem_client.repositories.variables.VariablesRepository) – Repository for project variables. +- [**VariablesRepository**](#cmem_client.repositories.variables.VariablesRepository) – Repository for project variables, execution variables and global variables. ## `VariableCreateConfig` {#cmem_client.repositories.variables.VariableCreateConfig} @@ -80,29 +98,41 @@ Variable update configuration. Bases: [Repository](../../repositories/base/abc/index.md#cmem_client.repositories.base.abc.Repository), [DeleteItemProtocol](../../repositories/protocols/delete_item/index.md#cmem_client.repositories.protocols.delete_item.DeleteItemProtocol), [CreateItemProtocol](../../repositories/protocols/create_item/index.md#cmem_client.repositories.protocols.create_item.CreateItemProtocol), [UpdateItemProtocol](../../repositories/protocols/update_item/index.md#cmem_client.repositories.protocols.update_item.UpdateItemProtocol) -Repository for project variables. +Repository for project variables, execution variables and global variables. + +Manages the variables of all projects in the Corporate Memory DataIntegration +(Build) environment together with the execution variables of their tasks. +Project variables are stored with keys in the form ``project_id:variable_name``, +execution variables with keys in the form ``project_id:task_id:variable_name``. + +The global variables are part of the repository as well, with their name as key. +They are defined in the configuration of DataIntegration, so they are read-only +and can neither be created, updated nor deleted. -Manages project variables across all projects in the Corporate Memory -DataIntegration (build) environment. Variables are fetched per project and -stored with combined keys in the form ``project_id:variable_name``. +Fetching all variables masks sensitive ones, their ``value`` is empty. Use +``get_item()`` to read a single project or execution variable including its value. + +**Attributes:** + +- [**tasks**](#cmem_client.repositories.variables.VariablesRepository.tasks) (dict[str, str]) – Labels of all tasks of all projects by ``project_id:task_id``, including +tasks without execution variables, as of the last fetch. +- [**errors**](#cmem_client.repositories.variables.VariablesRepository.errors) (dict[str, str]) – Errors the server reported for single variables on the last fetch, by +variable ID. A variable has an error, e.g. if its template can not be resolved. **Functions:** - [**create_item**](#cmem_client.repositories.variables.VariablesRepository.create_item) – Create (add) a new item to the repository - [**delete_all**](#cmem_client.repositories.variables.VariablesRepository.delete_all) – Delete all items from the repository - [**delete_item**](#cmem_client.repositories.variables.VariablesRepository.delete_item) – Delete an item from the repository -- [**fetch_data**](#cmem_client.repositories.variables.VariablesRepository.fetch_data) – Fetch all variables from all projects. -- [**get_item**](#cmem_client.repositories.variables.VariablesRepository.get_item) – Get a single variable by project and name. +- [**fetch_data**](#cmem_client.repositories.variables.VariablesRepository.fetch_data) – Fetch the global variables, those of all projects and the execution variables of their tasks. +- [**get_dependents**](#cmem_client.repositories.variables.VariablesRepository.get_dependents) – Get the variables and tasks which still use a variable. +- [**get_item**](#cmem_client.repositories.variables.VariablesRepository.get_item) – Get a single variable by project, name and optionally task. - [**items**](#cmem_client.repositories.variables.VariablesRepository.items) – Get the items of the repository - [**keys**](#cmem_client.repositories.variables.VariablesRepository.keys) – Get the keys of the repository - [**raise_modification_error**](#cmem_client.repositories.variables.VariablesRepository.raise_modification_error) – Raise an exception if needed - [**update_item**](#cmem_client.repositories.variables.VariablesRepository.update_item) – Update an existing item in the repository. - [**values**](#cmem_client.repositories.variables.VariablesRepository.values) – Get the values of the repository -**Attributes:** - -- [**logger**](#cmem_client.repositories.variables.VariablesRepository.logger) (Logger) – Gets the client logger - ### `create_item` {#cmem_client.repositories.variables.VariablesRepository.create_item} ```python @@ -149,26 +179,62 @@ Delete an item from the repository - [RepositoryModificationError](../../exceptions/index.md#cmem_client.exceptions.RepositoryModificationError) – if an error occurs while creating the item - HTTPError – for any other http error +### `errors` {#cmem_client.repositories.variables.VariablesRepository.errors} + +```python +errors: dict[str, str] +``` + ### `fetch_data` {#cmem_client.repositories.variables.VariablesRepository.fetch_data} ```python fetch_data() ``` -Fetch all variables from all projects. +Fetch the global variables, those of all projects and the execution variables of their tasks. + +**Raises:** + +- [RepositoryReadError](../../exceptions/index.md#cmem_client.exceptions.RepositoryReadError) – If Build does not provide the variables in one request, +because it is too old. + +### `get_dependents` {#cmem_client.repositories.variables.VariablesRepository.get_dependents} + +```python +get_dependents(key) +``` + +Get the variables and tasks which still use a variable. + +A variable which is used cannot be deleted. The server reports only some of +its dependents on a failed deletion, while this lists all of them. + +**Parameters:** + +- **key** (str) – Combined variable key in the format ``project_id:variable_name`` or +``project_id:task_id:variable_name``. + +**Returns:** + +- [VariableDependents](../../models/variable/index.md#cmem_client.models.variable.VariableDependents) – The variables and tasks which use the variable. + +**Raises:** + +- HTTPStatusError – If the variable is not found or the request fails. ### `get_item` {#cmem_client.repositories.variables.VariablesRepository.get_item} ```python -get_item(project_id, variable_name) +get_item(project_id, variable_name, task_id=None) ``` -Get a single variable by project and name. +Get a single variable by project, name and optionally task. **Parameters:** - **project_id** (str) – The project ID. - **variable_name** (str) – The variable name. +- **task_id** (str | None) – The task ID of an execution variable. Omit it for a project variable. **Returns:** @@ -210,6 +276,12 @@ raise_modification_error(response) Raise an exception if needed +### `tasks` {#cmem_client.repositories.variables.VariablesRepository.tasks} + +```python +tasks: dict[str, str] +``` + ### `update_item` {#cmem_client.repositories.variables.VariablesRepository.update_item} ```python diff --git a/docs/develop/cmem-client-api/api-reference/repositories/workflows/index.md b/docs/develop/cmem-client-api/api-reference/repositories/workflows/index.md index 6fd1bf4f0..11197ec99 100644 --- a/docs/develop/cmem-client-api/api-reference/repositories/workflows/index.md +++ b/docs/develop/cmem-client-api/api-reference/repositories/workflows/index.md @@ -14,7 +14,8 @@ tags: Repository for the workflows of DataIntegration projects. Provides WorkflowsRepository for listing workflows and for starting them, polling -their execution status and running them with input or output payloads. +their execution status, running them with input or output payloads and reading their +execution reports. **Examples:** @@ -41,6 +42,13 @@ Look at the status of every workflow currently known: >>> client.workflows.get_all_statuses() ``` +List the execution reports of a workflow and fetch the latest one: + +```pycon +>>> reports = client.workflows.list_reports("my-project:my-workflow") +>>> client.workflows.get_report("my-project:my-workflow", reports[0].time) +``` + **Classes:** - [**WorkflowsRepository**](#cmem_client.repositories.workflows.WorkflowsRepository) – Repository for managing workflows in Corporate Memory. @@ -51,8 +59,9 @@ Bases: [TaskSearchRepository](../../repositories/base/task_search/index.md Repository for managing workflows in Corporate Memory. -The dict (keys, values, items) is populated via the task search API which returns -workflows with io info (variableInputs/variableOutputs) and tags in a single call. +The dict (keys, values, items) is populated via the task search API, which returns +the workflows with their tags in a single call, and the workflow info API, which +resolves their variable inputs and outputs (variableInputs/variableOutputs). Operational methods (execute, get_status, execute_io, etc.) are independent of the dict and hit dedicated activity/result endpoints. @@ -61,13 +70,18 @@ the dict and hit dedicated activity/result endpoints. - [**execute**](#cmem_client.repositories.workflows.WorkflowsRepository.execute) – Execute the workflow without waiting for completion. - [**execute_io**](#cmem_client.repositories.workflows.WorkflowsRepository.execute_io) – Execute a workflow with variable input/output as a streaming context manager. - [**execute_wait_for_completion**](#cmem_client.repositories.workflows.WorkflowsRepository.execute_wait_for_completion) – Execute the workflow and block until it finishes. -- [**fetch_data**](#cmem_client.repositories.workflows.WorkflowsRepository.fetch_data) – Fetch a list from the DI task search endpoint via a type adapter. +- [**execute_with_variables**](#cmem_client.repositories.workflows.WorkflowsRepository.execute_with_variables) – Execute the workflow once with overridden execution variables. +- [**fetch_data**](#cmem_client.repositories.workflows.WorkflowsRepository.fetch_data) – Fetch the workflows and complete their variable inputs and outputs. - [**get_all_statuses**](#cmem_client.repositories.workflows.WorkflowsRepository.get_all_statuses) – Get status information for multiple workflow activities. +- [**get_execution_variables**](#cmem_client.repositories.workflows.WorkflowsRepository.get_execution_variables) – Get the execution variables a run of the workflow needs. +- [**get_report**](#cmem_client.repositories.workflows.WorkflowsRepository.get_report) – Get one execution report of a workflow. - [**get_status**](#cmem_client.repositories.workflows.WorkflowsRepository.get_status) – Get the current status of a workflow activity. - [**get_task**](#cmem_client.repositories.workflows.WorkflowsRepository.get_task) – Get full task details from the API. - [**get_workflow_editor_url**](#cmem_client.repositories.workflows.WorkflowsRepository.get_workflow_editor_url) – Get the URL to open a workflow in the workbench editor. - [**items**](#cmem_client.repositories.workflows.WorkflowsRepository.items) – Get the items of the repository - [**keys**](#cmem_client.repositories.workflows.WorkflowsRepository.keys) – Get the keys of the repository +- [**list_reports**](#cmem_client.repositories.workflows.WorkflowsRepository.list_reports) – List the execution reports available for a workflow. +- [**remove_execution**](#cmem_client.repositories.workflows.WorkflowsRepository.remove_execution) – Remove a finished run started with ``execute_with_variables()``. - [**values**](#cmem_client.repositories.workflows.WorkflowsRepository.values) – Get the values of the repository **Attributes:** @@ -94,7 +108,7 @@ Execute the workflow without waiting for completion. ### `execute_io` {#cmem_client.repositories.workflows.WorkflowsRepository.execute_io} ```python -execute_io(workflow_id, input_file=None, input_mime_type='application/xml', output_mime_type='application/xml', auto_config=False) +execute_io(workflow_id, input_file=None, input_mime_type='application/xml', output_mime_type='application/xml', auto_config=False, execution_variables=None) ``` Execute a workflow with variable input/output as a streaming context manager. @@ -106,6 +120,9 @@ Execute a workflow with variable input/output as a streaming context manager. - **input_mime_type** (str) – MIME type of the input file. - **output_mime_type** (str) – MIME type expected for the output. - **auto_config** (bool) – Whether to enable auto-configuration of input datasets. +- **execution_variables** (dict[str, str] | None) – Values by variable name, overriding the defaults of the +workflow's execution variables for this run only. They are never part +of the input entity. **Yields:** @@ -133,13 +150,54 @@ Execute the workflow and block until it finishes. - [WorkflowExecutionError](../../exceptions/index.md#cmem_client.exceptions.WorkflowExecutionError) – If workflow execution failed. +### `execute_with_variables` {#cmem_client.repositories.workflows.WorkflowsRepository.execute_with_variables} + +```python +execute_with_variables(workflow_id, execution_variables) +``` + +Execute the workflow once with overridden execution variables. + +The given values replace the defaults of the workflow's execution variables for +this run only, the stored defaults stay unchanged. The run is started as an +instance of the ``ExecuteWorkflowWithPayload`` activity, so poll its status with +``get_status()`` and that activity name and instance ID. The server keeps a +limited number of these instances per workflow, so remove a finished one with +``remove_execution()``. + +The server neither rejects names which the workflow does not use nor checks +for missing required variables, so compare the values with +``get_execution_variables()`` beforehand if needed. + +**Parameters:** + +- **workflow_id** (str) – The workflow to execute (in the form of 'project_id:workflow_id') +- **execution_variables** (dict[str, str]) – Values by variable name, overriding the defaults. + +**Returns:** + +- str – The ID of the started activity instance. + +**Raises:** + +- [WorkflowExecutionError](../../exceptions/index.md#cmem_client.exceptions.WorkflowExecutionError) – If the workflow could not be started. + ### `fetch_data` {#cmem_client.repositories.workflows.WorkflowsRepository.fetch_data} ```python fetch_data() ``` -Fetch a list from the DI task search endpoint via a type adapter. +Fetch the workflows and complete their variable inputs and outputs. + +The task search only carries the replaceable datasets marked on a workflow +itself, so a workflow of workflows whose variable input or output sits in a +nested workflow would report none. The workflow info endpoint resolves nested +workflows as well, so its answer replaces the io information of every workflow. + +**Raises:** + +- [WorkflowReadError](../../exceptions/index.md#cmem_client.exceptions.WorkflowReadError) – If the workflow info request fails. ### `get_all_statuses` {#cmem_client.repositories.workflows.WorkflowsRepository.get_all_statuses} @@ -163,10 +221,58 @@ Get status information for multiple workflow activities. - [WorkflowReadError](../../exceptions/index.md#cmem_client.exceptions.WorkflowReadError) – If the status fetch request fails. +### `get_execution_variables` {#cmem_client.repositories.workflows.WorkflowsRepository.get_execution_variables} + +```python +get_execution_variables(workflow_id) +``` + +Get the execution variables a run of the workflow needs. + +This covers the variables used at execution time by the workflow's tasks and +sub-workflows, recursively, and the variables defined on the workflow itself. +Only the defaults defined on the workflow apply to a run, so a variable which is +used but neither has such a default nor is set by a preceding node is marked as +required. + +**Parameters:** + +- **workflow_id** (str) – Workflow ID in the form 'project_id:workflow_id'. + +**Returns:** + +- list[[WorkflowExecutionVariable](../../models/variable/index.md#cmem_client.models.variable.WorkflowExecutionVariable)] – The execution variables, sorted by name. + +**Raises:** + +- [WorkflowReadError](../../exceptions/index.md#cmem_client.exceptions.WorkflowReadError) – If the workflow does not exist or the request fails. + +### `get_report` {#cmem_client.repositories.workflows.WorkflowsRepository.get_report} + +```python +get_report(workflow_id, time) +``` + +Get one execution report of a workflow. + +**Parameters:** + +- **workflow_id** (str) – Workflow ID in the form 'project_id:task_id'. +- **time** (datetime) – Timestamp of the report, as returned by +[list_reports](#cmem_client.repositories.workflows.WorkflowsRepository.list_reports). + +**Returns:** + +- [WorkflowReport](../../models/workflow/index.md#cmem_client.models.workflow.WorkflowReport) – The execution report. + +**Raises:** + +- [WorkflowReadError](../../exceptions/index.md#cmem_client.exceptions.WorkflowReadError) – If the report could not be fetched. + ### `get_status` {#cmem_client.repositories.workflows.WorkflowsRepository.get_status} ```python -get_status(workflow_id, activity_name='ExecuteDefaultWorkflow') +get_status(workflow_id, activity_name='ExecuteDefaultWorkflow', instance_id=None) ``` Get the current status of a workflow activity. @@ -175,6 +281,8 @@ Get the current status of a workflow activity. - **workflow_id** (str) – Workflow ID in the form 'project_id:workflow_id'. - **activity_name** ([ACTIVITY_NAME](../../models/workflow/index.md#cmem_client.models.workflow.ACTIVITY_NAME)) – Activity to check. Defaults to "ExecuteDefaultWorkflow". +- **instance_id** (str | None) – Activity instance to check, e.g. the one returned by +``execute_with_variables()``. Defaults to the main instance. **Returns:** @@ -234,6 +342,26 @@ keys() Get the keys of the repository +### `list_reports` {#cmem_client.repositories.workflows.WorkflowsRepository.list_reports} + +```python +list_reports(workflow_id) +``` + +List the execution reports available for a workflow. + +**Parameters:** + +- **workflow_id** (str) – Workflow ID in the form 'project_id:task_id'. + +**Returns:** + +- list[[WorkflowReportInfo](../../models/workflow/index.md#cmem_client.models.workflow.WorkflowReportInfo)] – The reports available for the workflow. Empty if it was never executed. + +**Raises:** + +- [WorkflowReadError](../../exceptions/index.md#cmem_client.exceptions.WorkflowReadError) – If the report list could not be fetched. + ### `logger` {#cmem_client.repositories.workflows.WorkflowsRepository.logger} ```python @@ -242,6 +370,23 @@ logger: logging.Logger Gets the client logger +### `remove_execution` {#cmem_client.repositories.workflows.WorkflowsRepository.remove_execution} + +```python +remove_execution(workflow_id, instance_id) +``` + +Remove a finished run started with ``execute_with_variables()``. + +**Parameters:** + +- **workflow_id** (str) – Workflow ID in the form 'project_id:workflow_id'. +- **instance_id** (str) – The activity instance ID of the run. + +**Raises:** + +- [WorkflowExecutionError](../../exceptions/index.md#cmem_client.exceptions.WorkflowExecutionError) – If the request fails. + ### `values` {#cmem_client.repositories.workflows.WorkflowsRepository.values} ```python diff --git a/docs/develop/cmem-client-api/api-reference/repositories/workspace_configs/index.md b/docs/develop/cmem-client-api/api-reference/repositories/workspace_configs/index.md index 05543e166..4ca1d310a 100644 --- a/docs/develop/cmem-client-api/api-reference/repositories/workspace_configs/index.md +++ b/docs/develop/cmem-client-api/api-reference/repositories/workspace_configs/index.md @@ -214,7 +214,7 @@ Export an item from the repository to a file path. **Raises:** - [RepositoryItemNotFoundError](../../exceptions/index.md#cmem_client.exceptions.RepositoryItemNotFoundError) – If the specified item key is not found. -- [RepositoryReadError](../../exceptions/index.md#cmem_client.exceptions.RepositoryReadError) – If there's an error during export or path mismatch. +- [RepositoryReadError](../../exceptions/index.md#cmem_client.exceptions.RepositoryReadError) – If there is an error during export or path mismatch. ### `fetch_data` {#cmem_client.repositories.workspace_configs.WorkspaceConfigsRepository.fetch_data} diff --git a/docs/develop/cmemc-scripts/index.md b/docs/develop/cmemc-scripts/index.md index 034396a9a..1af35b446 100644 --- a/docs/develop/cmemc-scripts/index.md +++ b/docs/develop/cmemc-scripts/index.md @@ -90,7 +90,7 @@ for workflow in cmemc(command): ## Using a shebang to create an executable file -As a nice addon, you could extend your script with a [shebang](https://en.wikipedia.org/wiki/Shebang_(Unix)) to start [uv](https://docs.astral.sh/uv/). +As a nice add on, you could extend your script with a [shebang](https://en.wikipedia.org/wiki/Shebang_(Unix)) to start [uv](https://docs.astral.sh/uv/). This will also install and manage dependencies and python versions for you: The complete script looks like this: diff --git a/docs/develop/index.md b/docs/develop/index.md index 496787e8c..248ed9d81 100644 --- a/docs/develop/index.md +++ b/docs/develop/index.md @@ -13,7 +13,7 @@ Extend eccenca Corporate Memory with custom plugins and packages, and access it --- - [Accessing Graphs with Java Applications](accessing-graphs-with-java-applications/index.md) covers how to connect to Corporate Memory using a Java program. + [Accessing Graphs with Java Applications](accessing-graphs-with-java-applications/index.md) covers how to connect to eccenca Corporate Memory using a Java program. - :material-language-python: Python diff --git a/docs/develop/marketplace-packages/development/index.md b/docs/develop/marketplace-packages/development/index.md index 18093040b..ec532ea16 100644 --- a/docs/develop/marketplace-packages/development/index.md +++ b/docs/develop/marketplace-packages/development/index.md @@ -9,7 +9,7 @@ tags: ## Introduction -Marketplace Packages are archives that bundle content, functionality, and configuration from Corporate Memory for sharing and reuse. +Marketplace Packages are archives that bundle content, functionality, and configuration from eccenca Corporate Memory for sharing and reuse. Each package has its own release cycle. Packages can be installed and uninstalled during runtime. diff --git a/docs/develop/marketplace-packages/development/tutorial/index.md b/docs/develop/marketplace-packages/development/tutorial/index.md index fdf4720b4..bd9cdbcda 100644 --- a/docs/develop/marketplace-packages/development/tutorial/index.md +++ b/docs/develop/marketplace-packages/development/tutorial/index.md @@ -43,7 +43,7 @@ You should now have a folder with two levels of files: The nested folder `my-package-id` represents your working directory for developing the package. -To add content to the package, simply copy the files you want to add into this folder, or extract existing content from a live Corporate Memory instance into the working directory. +To add content to the package, simply copy the files you want to add into this folder, or extract existing content from a live eccenca Corporate Memory instance into the working directory. !!! example "Extracting Corporate Memory content to add to the package" diff --git a/docs/develop/marketplace-packages/installation/index.md b/docs/develop/marketplace-packages/installation/index.md index 0da014ce3..fb66e6737 100644 --- a/docs/develop/marketplace-packages/installation/index.md +++ b/docs/develop/marketplace-packages/installation/index.md @@ -9,7 +9,7 @@ tags: ## Introduction -Marketplace Packages can be installed directly from a Corporate Memory Marketplace Server (e.g. [https://eccenca.market](https://eccenca.market)), or from local **C**orporate Memory **P**ackage **A**rchives (`.cpa` files) and package directories. +Marketplace Packages can be installed directly from the eccenca Corporate Memory Marketplace Server (e.g. [https://eccenca.market](https://eccenca.market)), or from local **C**orporate Memory **P**ackage **A**rchives (`.cpa` files) and package directories. This page describes how to search, install, list, and uninstall Marketplace Packages using `cmemc`. diff --git a/docs/develop/python-plugins/development/index.md b/docs/develop/python-plugins/development/index.md index a1b95218b..9bbc56fbf 100644 --- a/docs/develop/python-plugins/development/index.md +++ b/docs/develop/python-plugins/development/index.md @@ -19,7 +19,7 @@ This page gives an overview of the concepts you need to understand in order to d ## Base Package -`cmem-plugin-base` is a Python library that provides a set of base classes for developing plugins for the eccenca Corporate Memory (CMEM) platform. These base classes provide a consistent interface for defining new plugins, handling configuration, and communicating with the Build (DataIntegration) of CMEM. +`cmem-plugin-base` is a Python library that provides a set of base classes for developing plugins for the eccenca Corporate Memory platform. These base classes provide a consistent interface for defining new plugins, handling configuration, and communicating with the Build (DataIntegration) of Corporate Memory. ## Plugin Types @@ -663,7 +663,7 @@ This JVM-based logger will prefix all plugin logs with `plugins.python. rdf:type di:Dataset`) indicating that this graph was written by DataIntegration. +- If the option `eccencaDataPlatform.writeGraphType` is set to true the a graph in Corporate Memory generated by Dataintegration will contain a triple (` rdf:type di:Dataset`) indicating that this graph was written by DataIntegration. - Improved mapping suggestion: - Allows to select from multiple matching candidates. - The user can switch between data source and target vocabulary view. diff --git a/docs/release-notes/2021/corporate-memory-21-04/index.md b/docs/release-notes/2021/corporate-memory-21-04/index.md index 6ccf41371..b691240c2 100644 --- a/docs/release-notes/2021/corporate-memory-21-04/index.md +++ b/docs/release-notes/2021/corporate-memory-21-04/index.md @@ -2,7 +2,7 @@ tags: - ReleaseNote --- -# Corporate Memory 21.04 +# eccenca Corporate Memory 21.04 Corporate Memory 21.04 is the second release in 2021. diff --git a/docs/release-notes/2021/corporate-memory-21-06/index.md b/docs/release-notes/2021/corporate-memory-21-06/index.md index aab44042f..9612b938c 100644 --- a/docs/release-notes/2021/corporate-memory-21-06/index.md +++ b/docs/release-notes/2021/corporate-memory-21-06/index.md @@ -2,7 +2,7 @@ tags: - ReleaseNote --- -# Corporate Memory 21.06 +# eccenca Corporate Memory 21.06 Corporate Memory 21.06 is the third release in 2021. diff --git a/docs/release-notes/2021/corporate-memory-21-11/index.md b/docs/release-notes/2021/corporate-memory-21-11/index.md index 9e314fd42..287fca0f9 100644 --- a/docs/release-notes/2021/corporate-memory-21-11/index.md +++ b/docs/release-notes/2021/corporate-memory-21-11/index.md @@ -2,7 +2,7 @@ tags: - ReleaseNote --- -# Corporate Memory 21.11 +# eccenca Corporate Memory 21.11 Corporate Memory 21.11 is the fourth release in 2021. diff --git a/docs/release-notes/2022/corporate-memory-22-1/index.md b/docs/release-notes/2022/corporate-memory-22-1/index.md index 1552a910b..6ce6d83e6 100644 --- a/docs/release-notes/2022/corporate-memory-22-1/index.md +++ b/docs/release-notes/2022/corporate-memory-22-1/index.md @@ -2,7 +2,7 @@ tags: - ReleaseNote --- -# Corporate Memory 22.1 +# eccenca Corporate Memory 22.1 Corporate Memory 22.1 is the first release in 2022. @@ -212,7 +212,7 @@ In addition to that, multiple performance and stability issues were solved. ### DataManager - New Graph List component with configurable lists. - - The lists can be configured in the cmem config graph. + - The lists can be configured in the Corporate Memory config graph. - Configurations in the `/application.yml` are ignored any customizations need to be migrated. ### DataPlatform diff --git a/docs/release-notes/2022/corporate-memory-22-2/index.md b/docs/release-notes/2022/corporate-memory-22-2/index.md index 044580b40..0d56d9181 100644 --- a/docs/release-notes/2022/corporate-memory-22-2/index.md +++ b/docs/release-notes/2022/corporate-memory-22-2/index.md @@ -2,7 +2,7 @@ tags: - ReleaseNote --- -# Corporate Memory 22.2.3 +# eccenca Corporate Memory 22.2.3 Corporate Memory 22.2.3 is the third patch release in the 22.2 release line. diff --git a/docs/release-notes/2023/corporate-memory-23-1/index.md b/docs/release-notes/2023/corporate-memory-23-1/index.md index a8eaaee63..6f83b6623 100644 --- a/docs/release-notes/2023/corporate-memory-23-1/index.md +++ b/docs/release-notes/2023/corporate-memory-23-1/index.md @@ -2,7 +2,7 @@ tags: - ReleaseNote --- -# Corporate Memory 23.1.3 +# eccenca Corporate Memory 23.1.3 Corporate Memory 23.1.3 is the second patch release in the 23.1 release line. diff --git a/docs/release-notes/2023/corporate-memory-23-2/index.md b/docs/release-notes/2023/corporate-memory-23-2/index.md index 7da431060..89a1e63d5 100644 --- a/docs/release-notes/2023/corporate-memory-23-2/index.md +++ b/docs/release-notes/2023/corporate-memory-23-2/index.md @@ -2,7 +2,7 @@ tags: - ReleaseNote --- -# Corporate Memory 23.2.1 +# eccenca Corporate Memory 23.2.1 Corporate Memory 23.2.1 is the first patch release in the 23.2 release line. diff --git a/docs/release-notes/2023/corporate-memory-23-3/index.md b/docs/release-notes/2023/corporate-memory-23-3/index.md index 31e39f5ed..2816058bc 100644 --- a/docs/release-notes/2023/corporate-memory-23-3/index.md +++ b/docs/release-notes/2023/corporate-memory-23-3/index.md @@ -2,7 +2,7 @@ tags: - ReleaseNote --- -# Corporate Memory 23.3.2 +# eccenca Corporate Memory 23.3.2 Corporate Memory 23.3.2 is the second patch release in the 23.3 release line. @@ -124,7 +124,7 @@ v23.3.0 of DataManager ships the following changes: v23.3.0 of DataManager ships the following fixes: - Fixed broken navigation (workspace part of URL was lost). -- CMEM Manual Testing 23.2 e2e - Don't do redundant redirects in the Module context. +- Manual Testing 23.2 e2e - Don't do redundant redirects in the Module context. - Navigation tree in the Thesaurus module was collapsed after a subitem select. - Use more space for visualization catalogue if available. diff --git a/docs/release-notes/2024/corporate-memory-24-1/index.md b/docs/release-notes/2024/corporate-memory-24-1/index.md index 99a37f5fa..c36a47fbf 100644 --- a/docs/release-notes/2024/corporate-memory-24-1/index.md +++ b/docs/release-notes/2024/corporate-memory-24-1/index.md @@ -2,7 +2,7 @@ tags: - ReleaseNote --- -# Corporate Memory 24.1.3 +# eccenca Corporate Memory 24.1.3 Corporate Memory 24.1.3 is the third patch release in the 24.1 release line. diff --git a/docs/release-notes/2024/corporate-memory-24-2/index.md b/docs/release-notes/2024/corporate-memory-24-2/index.md index 4495d8466..461c6c53e 100644 --- a/docs/release-notes/2024/corporate-memory-24-2/index.md +++ b/docs/release-notes/2024/corporate-memory-24-2/index.md @@ -2,7 +2,7 @@ tags: - ReleaseNote --- -# Corporate Memory 24.2.1 +# eccenca Corporate Memory 24.2.1 Corporate Memory 24.2.1 is the first patch release in the 24.2 release line. diff --git a/docs/release-notes/2024/corporate-memory-24-3/index.md b/docs/release-notes/2024/corporate-memory-24-3/index.md index 8de9c28db..3f591ecfd 100644 --- a/docs/release-notes/2024/corporate-memory-24-3/index.md +++ b/docs/release-notes/2024/corporate-memory-24-3/index.md @@ -2,7 +2,7 @@ tags: - ReleaseNote --- -# Corporate Memory 24.3.2 +# eccenca Corporate Memory 24.3.2 Corporate Memory 24.3.2 is the third major release in 2024. diff --git a/docs/release-notes/2025/corporate-memory-25-1/index.md b/docs/release-notes/2025/corporate-memory-25-1/index.md index c799f8494..3f9e52582 100644 --- a/docs/release-notes/2025/corporate-memory-25-1/index.md +++ b/docs/release-notes/2025/corporate-memory-25-1/index.md @@ -2,7 +2,7 @@ tags: - ReleaseNote --- -# Corporate Memory 25.1.2 +# eccenca Corporate Memory 25.1.2 Corporate Memory 25.1 is the first major release in 2025. diff --git a/docs/release-notes/2025/corporate-memory-25-2/index.md b/docs/release-notes/2025/corporate-memory-25-2/index.md index abcb924e5..a00137d68 100644 --- a/docs/release-notes/2025/corporate-memory-25-2/index.md +++ b/docs/release-notes/2025/corporate-memory-25-2/index.md @@ -2,7 +2,7 @@ tags: - ReleaseNote --- -# Corporate Memory 25.2.7 +# eccenca Corporate Memory 25.2.7 Corporate Memory 25.2 is the second major release in 2025. @@ -126,7 +126,7 @@ We are pleased to announce Explore v25.2.6, which brings significant enhancement **v25.2.1 of Explore ships the following fixes and additions:** -- Pagination for inline views of Dataset (used for preview in cmem build) +- Pagination for inline views of Dataset (used for preview in Build workspace) - Widgets for shaped resources without properties to display are shown - Update/Replace in the explore graph list diff --git a/docs/release-notes/2025/corporate-memory-25-3/index.md b/docs/release-notes/2025/corporate-memory-25-3/index.md index bff7eac25..f154cc07c 100644 --- a/docs/release-notes/2025/corporate-memory-25-3/index.md +++ b/docs/release-notes/2025/corporate-memory-25-3/index.md @@ -3,7 +3,7 @@ tags: - ReleaseNote --- -# Corporate Memory 25.3.4 +# eccenca Corporate Memory 25.3.4 Corporate Memory 25.3 is the third major release in 2025. diff --git a/docs/release-notes/2026/corporate-memory-26-1/index.md b/docs/release-notes/2026/corporate-memory-26-1/index.md index 7c4facb96..aa817e9d9 100644 --- a/docs/release-notes/2026/corporate-memory-26-1/index.md +++ b/docs/release-notes/2026/corporate-memory-26-1/index.md @@ -3,7 +3,7 @@ tags: - ReleaseNote --- -# Corporate Memory 26.1.3 +# eccenca Corporate Memory 26.1.3 Corporate Memory 26.1 is the first major release in 2026. It expands AI-assisted mapping in Build, refreshes the resource experience in Explore, and strengthens access-condition and query-catalog administration in cmemc. diff --git a/docs/training/ontology-modeling/index.md b/docs/training/ontology-modeling/index.md index 062d2e1fd..8a5a72c8d 100644 --- a/docs/training/ontology-modeling/index.md +++ b/docs/training/ontology-modeling/index.md @@ -17,7 +17,7 @@ tags: This module introduces RDF Schema and OWL, the two W3C languages for describing the classes and properties of a domain. It builds on [RDF and Linked Data Foundations](../rdf-and-linked-data/index.md) and [SPARQL Basics](../sparql-basics/index.md), and prepares the hands-on module [Ontology Engineering](../ontology-engineering/index.md). - It is part of the [Corporate Memory training](../index.md). + It is part of the [eccenca Corporate Memory training](../index.md). Use **Show slide** next to a heading to bring the matching slide into view, or **Read the explanation** under the slides to jump to the text for the current slide. diff --git a/docs/training/sparql-basics/index.md b/docs/training/sparql-basics/index.md index 2a045f34c..607226591 100644 --- a/docs/training/sparql-basics/index.md +++ b/docs/training/sparql-basics/index.md @@ -16,10 +16,10 @@ tags: !!! abstract "About this module" This module teaches how to query RDF graphs with SPARQL, from the first triple pattern to aggregation and updates, and ends with exercises on the product Knowledge Graph. - It builds on [RDF and Linked Data Foundations](../rdf-and-linked-data/index.md) and is part of the [Corporate Memory training](../index.md). + It builds on [RDF and Linked Data Foundations](../rdf-and-linked-data/index.md) and is part of the [eccenca Corporate Memory training](../index.md). Use **Show slide** next to a heading to bring the matching slide into view, or **Read the explanation** under the slides to jump to the text for the current slide. - All queries run in the [Query module](../../explore-and-author/query-module/index.md) of eccenca Corporate Memory. + All queries run in the [Query module](../../explore-and-author/query-module/index.md) of Corporate Memory. The **Try it** boxes run the queries of the slides directly in the browser, against the data shown in the box, without a Corporate Memory instance. ## Why a query language for RDF { #why-a-query-language } diff --git a/lychee.toml b/lychee.toml new file mode 100644 index 000000000..48e5f41db --- /dev/null +++ b/lychee.toml @@ -0,0 +1,77 @@ +# lychee configuration for `task check:links` - https://lychee.cli.rs/ +# +# Checks the outgoing http(s) links of the Markdown sources under docs/. +# Links inside the site are checked by `zensical build --strict` and +# `task check:build-output`, not here. + +# Outgoing links only. +scheme = ["https", "http"] + +# Several sites (w3.org, salesforce.com, ...) answer 403 to non-browser +# clients, so present a browser user agent. +user_agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0 Safari/537.36" + +# 429 means "rate limited", not "gone". +accept = ["100..=103", "200..=299", "429"] + +# A run lasts as long as its slowest link: an unreachable host costs +# timeout * (max_retries + 1) plus the waits, on every run (failures are not cached). +# 20s leaves room for slow but working pages (the Microsoft download pages take ~9s). +timeout = 20 +max_retries = 2 +retry_wait_time = 1 +max_concurrency = 32 + +# Remember results for a day, so repeated local runs do not hammer the same hosts. +cache = true +max_cache_age = "1d" + +# Generated configuration reference: its prose holds `PROTOCOL://HOST:PORT/CONTEXT_PATH`, +# which lychee reports as an unparsable URL before any exclude pattern applies. +exclude_path = [ + 'docs/deploy-and-configure/configuration/explore/dataplatform/application-full\.md', +] + +exclude = [ + # Placeholder hosts and values in examples and configuration defaults. + '^https?://localhost', + '^https?://127\.0\.0\.1', + '^https?://([a-z0-9-]+\.)*example\.(com|org|net)', + '^https?://remote:', + '^https?://my-cmem-system', + '^https?://fallback\.eccenca\.com', + '^https?://cmem-mcp-server\.eccenca\.com', + '^https?://database\.windows\.net', + '^https?://ld\.company\.org', + '^https?://namespace\.org', + '^PROTOCOL://', + + # IRIs: identifiers written as autolinks, not documents to fetch. + '^https?://vocab\.eccenca\.com/', + '^https?://ns\.eccenca\.com/', + '^https?://attack\.mitre\.org/inferences', + '^https?://rdfs\.org/ns/', + + # Sites that answer 403 to any automated client, whatever the user agent. + '^https?://(www\.)?w3\.org/', + '^https?://developer\.salesforce\.com/', + '^https?://jfact\.sourceforge\.net', + '^https?://(www\.)?wikiwand\.com/', + '^https?://(www\.)?npmjs\.com/', + # Answers 403 to GitHub Actions runners (works locally, fails in CI). + '^https?://(www\.)?(x|twitter)\.com/', + + # Behind a login. + '^https?://releases\.eccenca\.com/', + + # Known failures, excluded so the gate stays usable. Check each again and remove + # the entry once the link works or the page no longer has it. + # 2026-11-12: 404; in generated plugin descriptions (Geo transformers, SearchAddresses) - needs a ticket. + '^https://adresse\.data\.gouv\.fr/$', + # 2026-11-12: TLS certificate expired; in the generated ShapesPlugin reference. + '^https://prefix\.cc/', + # 2026-11-12: host refuses connections (www.openlinksw.com too); system-architecture page. + '^https://virtuoso\.openlinksw\.com/', + # 2026-11-12: times out for automated clients; GNU make bug link on two installation pages. + '^https://savannah\.gnu\.org/', +] diff --git a/nav.yml b/nav.yml index 0499e87bc..7de7ba612 100644 --- a/nav.yml +++ b/nav.yml @@ -95,9 +95,11 @@ nav: - Download Git files: build/reference/customtask/cmem_plugin_git-Download.md - Download Nextcloud files: build/reference/customtask/cmem_plugin_nextcloud-Download.md - Download Office 365 Files: build/reference/customtask/cmem_plugin_office365-Download.md + - Download S3 objects: build/reference/customtask/cmem_plugin_s3-Download.md - Download SSH files: build/reference/customtask/cmem_plugin_ssh-Download.md - Evaluate template: build/reference/customtask/Template.md - Execute a command in a kubernetes pod: build/reference/customtask/cmem_plugin_kubernetes-Execute.md + - Execute Catalog Query: build/reference/customtask/cmem_plugin_querycatalog-ExecuteCatalogQuery.md - Execute commands via SSH: build/reference/customtask/cmem_plugin_ssh-Execute.md - Execute Instructions: build/reference/customtask/cmem_plugin_llm-ExecuteInstructions.md - Execute REST requests: build/reference/customtask/eccencaRestOperator.md @@ -106,10 +108,15 @@ nav: - Generate base36 IRDIs: build/reference/customtask/cmem_plugin_irdi-workflow-irdi_plugin-IrdiPlugin.md - Generate random values: build/reference/customtask/cmem_plugin_random-GenerateEntities.md - Generate SHACL shapes from data: build/reference/customtask/cmem_plugin_shapes-plugin_shapes-ShapesPlugin.md + - Get Catalog Query: build/reference/customtask/cmem_plugin_querycatalog-GetCatalogQuery.md - Get emails: build/reference/customtask/cmem-plugin-email-GetEmails.md + - Get ntfy attachments: build/reference/customtask/cmem_plugin_ntfy-GetAttachments.md + - Get ntfy messages: build/reference/customtask/cmem_plugin_ntfy-GetMessages.md - Get project files: build/reference/customtask/getProjectFiles.md - Get workflow report: build/reference/customtask/cmem_plugin_wfreports_get_report.md - - GraphQL query: build/reference/customtask/cmem_plugin_graphql-workflow-graphql-GraphQLPlugin.md + - GraphQL query: build/reference/customtask/cmem_plugin_graphql-Query.md + - 'Hetzner Cloud: List Resources': build/reference/customtask/cmem_plugin_hetzner-ListResources.md + - 'Hetzner Cloud: List Server Types': build/reference/customtask/cmem_plugin_hetzner-ListServerTypes.md - Install Packages: build/reference/customtask/cmem_plugin_packages-InstallPackages.md - Join tables: build/reference/customtask/Merge.md - jq: build/reference/customtask/cmem-plugin-jq-workflow.md @@ -117,10 +124,12 @@ nav: - JSON to File: build/reference/customtask/jsonToFile.md - Kafka Consumer (Receive Messages): build/reference/customtask/cmem_plugin_kafka-ReceiveMessages.md - Kafka Producer (Send Messages): build/reference/customtask/cmem_plugin_kafka-SendMessages.md + - List Catalog Queries: build/reference/customtask/cmem_plugin_querycatalog-ListCatalogQueries.md - List Git files: build/reference/customtask/cmem_plugin_git-List.md - List Nextcloud files: build/reference/customtask/cmem_plugin_nextcloud-List.md - List Office 365 Files: build/reference/customtask/cmem_plugin_office365-List.md - List project files: build/reference/customtask/cmem_plugin_project_resources-List.md + - List S3 objects: build/reference/customtask/cmem_plugin_s3-List.md - List SSH files: build/reference/customtask/cmem_plugin_ssh-List.md - Merge tables: build/reference/customtask/MultiTableMerge.md - Normalize units of measurement: build/reference/customtask/ucumNormalizationTask.md @@ -137,10 +146,13 @@ nav: - Scheduler: build/reference/customtask/Scheduler.md - Search addresses: build/reference/customtask/SearchAddresses.md - Search for Logs: build/reference/customtask/cmem_plugin_logpoint-search_logs_task-RetrieveLogs.md + - Search LDAP directory: build/reference/customtask/cmem_plugin_ldap-Search.md - Search Vector Embeddings: build/reference/customtask/cmem_plugin_pgvector-Search.md - Select random entities: build/reference/customtask/cmem_plugin_random-SelectEntities.md - Send email: build/reference/customtask/SendEMail.md - Send Mattermost messages: build/reference/customtask/cmem_plugin_mattermost.md + - Send ntfy attachment: build/reference/customtask/cmem_plugin_ntfy-SendAttachment.md + - Send ntfy message: build/reference/customtask/cmem_plugin_ntfy-SendMessage.md - Set execution variable: build/reference/customtask/setExecutionVariableOperator.md - Set or Overwrite parameters: build/reference/customtask/cmem_plugin_parameters-ParametersPlugin.md - Set parameters: build/reference/customtask/setParameters.md @@ -159,6 +171,7 @@ nav: - Upload File to Knowledge Graph: build/reference/customtask/eccencaDataPlatformGraphStoreFileUploadOperator.md - Upload files to CKAN: build/reference/customtask/cmem_plugin_ckan-Upload.md - Upload files to Nextcloud: build/reference/customtask/cmem_plugin_nextcloud-Upload.md + - Upload files to S3: build/reference/customtask/cmem_plugin_s3-Upload.md - Upload Git files: build/reference/customtask/cmem_plugin_git-Upload.md - Upload local files: build/reference/customtask/cmem_plugin_project_resources-UploadLocalFiles.md - Upload SSH files: build/reference/customtask/cmem_plugin_ssh-Upload.md @@ -673,6 +686,8 @@ nav: - automate/cmemc-command-line-interface/command-reference/admin/acl/index.md - Client: - automate/cmemc-command-line-interface/command-reference/admin/client/index.md + - Logs: + - automate/cmemc-command-line-interface/command-reference/admin/logs/index.md - Metrics: - automate/cmemc-command-line-interface/command-reference/admin/metrics/index.md - Migration: @@ -705,6 +720,8 @@ nav: - automate/cmemc-command-line-interface/command-reference/project/index.md - File: - automate/cmemc-command-line-interface/command-reference/project/file/index.md + - Rule block: + - automate/cmemc-command-line-interface/command-reference/project/rule-block/index.md - Variable: - automate/cmemc-command-line-interface/command-reference/project/variable/index.md - Query: @@ -775,14 +792,28 @@ nav: - components: - deployment: - develop/cmem-client-api/api-reference/components/deployment/index.md - - graph_store: - - develop/cmem-client-api/api-reference/components/graph_store/index.md + - build: + - component: + - develop/cmem-client-api/api-reference/components/build/component/index.md + - logs: + - develop/cmem-client-api/api-reference/components/build/logs/index.md + - workspace: + - develop/cmem-client-api/api-reference/components/build/workspace/index.md + - explore: + - component: + - develop/cmem-client-api/api-reference/components/explore/component/index.md + - logs: + - develop/cmem-client-api/api-reference/components/explore/logs/index.md + - sparql: + - develop/cmem-client-api/api-reference/components/explore/sparql/index.md + - titles: + - develop/cmem-client-api/api-reference/components/explore/titles/index.md + - graphinsights: + - component: + - develop/cmem-client-api/api-reference/components/graphinsights/component/index.md - marketplace: - - develop/cmem-client-api/api-reference/components/marketplace/index.md - - sparql_wrapper: - - develop/cmem-client-api/api-reference/components/sparql_wrapper/index.md - - workspace: - - develop/cmem-client-api/api-reference/components/workspace/index.md + - component: + - develop/cmem-client-api/api-reference/components/marketplace/component/index.md - models: - access_condition: - develop/cmem-client-api/api-reference/models/access_condition/index.md @@ -796,6 +827,8 @@ nav: - develop/cmem-client-api/api-reference/models/dataset/index.md - error: - develop/cmem-client-api/api-reference/models/error/index.md + - explore: + - develop/cmem-client-api/api-reference/models/explore/index.md - graph: - develop/cmem-client-api/api-reference/models/graph/index.md - graph_import: @@ -808,10 +841,14 @@ nav: - develop/cmem-client-api/api-reference/models/keycloak_client/index.md - logging_config: - develop/cmem-client-api/api-reference/models/logging_config/index.md + - logs: + - develop/cmem-client-api/api-reference/models/logs/index.md - marshalling_plugins: - develop/cmem-client-api/api-reference/models/marshalling_plugins/index.md - package: - develop/cmem-client-api/api-reference/models/package/index.md + - paged_list: + - develop/cmem-client-api/api-reference/models/paged_list/index.md - project: - develop/cmem-client-api/api-reference/models/project/index.md - python_install: @@ -822,6 +859,8 @@ nav: - develop/cmem-client-api/api-reference/models/query_catalog/index.md - resource: - develop/cmem-client-api/api-reference/models/resource/index.md + - rule_block: + - develop/cmem-client-api/api-reference/models/rule_block/index.md - scheduler: - develop/cmem-client-api/api-reference/models/scheduler/index.md - status: @@ -859,8 +898,8 @@ nav: - develop/cmem-client-api/api-reference/repositories/files/index.md - graph_imports: - develop/cmem-client-api/api-reference/repositories/graph_imports/index.md - - graph_insights: - - develop/cmem-client-api/api-reference/repositories/graph_insights/index.md + - graph_snapshots: + - develop/cmem-client-api/api-reference/repositories/graph_snapshots/index.md - graphs: - develop/cmem-client-api/api-reference/repositories/graphs/index.md - marketplace_packages: @@ -871,6 +910,8 @@ nav: - develop/cmem-client-api/api-reference/repositories/python_packages/index.md - queries: - develop/cmem-client-api/api-reference/repositories/queries/index.md + - rule_blocks: + - develop/cmem-client-api/api-reference/repositories/rule_blocks/index.md - schedulers: - develop/cmem-client-api/api-reference/repositories/schedulers/index.md - user_accounts: diff --git a/poetry.lock b/poetry.lock index fe287df37..0ae30025b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,175 +1,5 @@ # This file is automatically @generated by Poetry 2.5.1 and should not be changed by hand. -[[package]] -name = "aiohappyeyeballs" -version = "2.7.1" -description = "Happy Eyeballs for asyncio" -optional = false -python-versions = ">=3.10" -groups = ["dev"] -files = [ - {file = "aiohappyeyeballs-2.7.1-py3-none-any.whl", hash = "sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472"}, - {file = "aiohappyeyeballs-2.7.1.tar.gz", hash = "sha256:065665c041c42a5938ed220bdcd7230f22527fbec085e1853d2402c8a3615d9d"}, -] - -[[package]] -name = "aiohttp" -version = "3.14.3" -description = "Async http client/server framework (asyncio)" -optional = false -python-versions = ">=3.10" -groups = ["dev"] -files = [ - {file = "aiohttp-3.14.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:eb0495d778817619273c108784292be161a924b9f5ae5cbbc70a2caa6838250b"}, - {file = "aiohttp-3.14.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c3c200cf9757edd785051dc699c7ecbec22110dbfcb3fefc7a9f9695eda8ea7a"}, - {file = "aiohttp-3.14.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd51ebf9d3a00c074df4ede271023f4d2dba289bcc740b88191872716014e3c5"}, - {file = "aiohttp-3.14.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:134ac5ddcf61c6fad984b9a5727d83492ada43d63471db20fb73042c13fca62f"}, - {file = "aiohttp-3.14.3-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:70c987b27534f9ae1a723f47ae921571d616da21d3208282bf4c52af5164ac43"}, - {file = "aiohttp-3.14.3-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1b59533861b70a2185c8f4f350f791f39d64358ef6944ce71c5240c9ec0982c9"}, - {file = "aiohttp-3.14.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1c5281acc88b92396f88c7e1e2748f8466689df22b80170e4f51efa712fb47a8"}, - {file = "aiohttp-3.14.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:48d67b87db6279c044760787eb01f6413032c2e6f3ba1cafaa492b1c8e578479"}, - {file = "aiohttp-3.14.3-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f53bcd52f585e1ac3e590d61434eb61f9a88c38df041b4ea126d97144344a77b"}, - {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0fdea2281997af69da84c77ffa6f5938a0285f21fb3887c249d67419ca865b3d"}, - {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:cda5fd5c95ad7a125a2e8464acc78b98b94c475a3780d6aa0aa157c93f470f4d"}, - {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6debfa7312ff9d4c124dc71d72e9a0a4b9e0879e48ba6fcb42bef5c3300289e2"}, - {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:f4e05329faa0ea1a404b37de4f034fd2c2defcca06a68dc6745e4e56c88e8a48"}, - {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:a3a8296e7ab5c295f53f1041487cb088e1480775aafbf7fe545d93b770a0f96f"}, - {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5373dc80ad1aa2fb9ad95c83f24eef418bbda3a61375f128e5b0192e4f3f9b32"}, - {file = "aiohttp-3.14.3-cp310-cp310-win32.whl", hash = "sha256:a3e22975f905b89a55a488c2a08f2fdb2186175349e917d48985cc468a3d4c6e"}, - {file = "aiohttp-3.14.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdd0e2834dce1a26c1bbe26464861e16bbe217042cbff619247c11594472518c"}, - {file = "aiohttp-3.14.3-cp310-cp310-win_arm64.whl", hash = "sha256:eac645b09bcfdf73df7536331f0678c1086ea250981118ddb5199e17ccef72bb"}, - {file = "aiohttp-3.14.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e568e14940c09955aa51f4e645b6daa18a581c5dcfcd73744dcc86a856e3ced3"}, - {file = "aiohttp-3.14.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:54cfcdee2770dac994417cbb0ee1f3eb0e7cb6b30c79bf44f2c02ff79ec5124a"}, - {file = "aiohttp-3.14.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:21c016079415ed3fd676963e9793700a566d85dbbd6bfc564b9b2d209147dcc8"}, - {file = "aiohttp-3.14.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d6088ec9894113802bddb3c09e974929aed2c7b3a8c456219b8aab4481f1a239"}, - {file = "aiohttp-3.14.3-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:16ea7e24c309fb7c0bbd505d149abe4fe4dccfb8db911db7dbec0921bc889a6f"}, - {file = "aiohttp-3.14.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:56f355e79f71aef2a85c80305cc915f894b170dba76de5fe84f6351939b83c06"}, - {file = "aiohttp-3.14.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:18c441d0a8fca6de8d1f546849b9f0ab20d435993e2c5b59562b2fae6be2f929"}, - {file = "aiohttp-3.14.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53e7b4ce82b54a8bcc71b3b67a5cbd177ca1d7f592cbc92cd38b7349f73482db"}, - {file = "aiohttp-3.14.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f55119f7bf25f49ed210f6096090715da24f2943c62102448915fde3c62877ce"}, - {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9aa6e61fdf20105c4144e755bd586008ff450791d67b1c8146fdc15959c4d51c"}, - {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ccd4893707b3e2a13e39c90d43cf80edf2e4d0457935bcc103bf2346214c3f15"}, - {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:b2466434105a4e03113c36ec775cc2ebe6676b62eae326fa670bb607ef788c1c"}, - {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ba59d59aba08ac02fc03b0c8983ccd5ee39a199d0552ce9e6d2b4845b34d59ae"}, - {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ed099d105449c4f9e84f24af203cd131349d4761d8813fa7e02c32e7128cd910"}, - {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:152516815ef926786a0b6ae2b8f1fd2e0c71582dee0b435636865316fd4891b7"}, - {file = "aiohttp-3.14.3-cp311-cp311-win32.whl", hash = "sha256:a4af35c443e0b1a1bd6a8af3f3485d7fda15c142751a00f3ff8090f0b93346fa"}, - {file = "aiohttp-3.14.3-cp311-cp311-win_amd64.whl", hash = "sha256:e1e74298bab6ee0d6e749ed4fd1901c7e604bdda32c03d787a2cc71c46d0433d"}, - {file = "aiohttp-3.14.3-cp311-cp311-win_arm64.whl", hash = "sha256:03cd2bde3d7f085b64e549c985f4bb928cad7e8ecf5323bfca320db548d81b39"}, - {file = "aiohttp-3.14.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:39aded8c7f3b935b54aab1d8d73c70ec0ee2d3ec3b943e0e86611bc150ba47f5"}, - {file = "aiohttp-3.14.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5bcb6ff3fdab1258a192679ff1a05d44f59626430aa05cd1a9d2447423599228"}, - {file = "aiohttp-3.14.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:617105e2c3018ee38d0c8ce5ee3c84f621a6d8b9f723202aacaff28449ca91ee"}, - {file = "aiohttp-3.14.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f631fe87a6f30df5fbe6d79640b25e4cffb38c31c7fb6f10871517b84b0f8c1a"}, - {file = "aiohttp-3.14.3-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a94dbaae5ae27bd849c93570669bff91e0510f33a80805738e3de72a7be0447b"}, - {file = "aiohttp-3.14.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8f2f1c4c032c7cedd7d8da6f54c97b70266c6570c3108d3fdffee7188bb70529"}, - {file = "aiohttp-3.14.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ea05e1f97ceea523942d9b2a7d7c0359d781d683d6b043f5943a602b14da4787"}, - {file = "aiohttp-3.14.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:543906c127fb1d929b95076db19b83fa2d46751006ff1e23b093aa5ac4d8db42"}, - {file = "aiohttp-3.14.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0a5ff2dfbb9ce645fa5b8ef3e02c6c0b9cc3f6030ff863d0c51fffc50cb5541b"}, - {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:041badb8f84396357c4d3ad26de6afd7a32b112f43d3c63045c0c8278cfd2043"}, - {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:530125ee1163c4219af35dc3aa1206e541e7b31b6efc1a3f93b70a136f65d427"}, - {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c8653fd547c93a61aadc612007790f5555cdd18946fa48cf45e26d8ea4ea473d"}, - {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:89176250f686cb9853c0fb7ead90e639e915b84a6f43eedc2a4e7ec21f1037f0"}, - {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3a26434dafe408229ff3403458ca58de24fb51936504decac49ce6755f77e59d"}, - {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d1558173930a5a8d3069cee5c92fc91c87c4dbcb099debbb3622053717145a19"}, - {file = "aiohttp-3.14.3-cp312-cp312-win32.whl", hash = "sha256:16100ad3ab8d649fdfbee87602d9d2dcdca9df0b9eda8a1b5fdc0d41f96da559"}, - {file = "aiohttp-3.14.3-cp312-cp312-win_amd64.whl", hash = "sha256:33a2d7c28d33797a2e99923dffa63f83d908a19b6bf26cfe80fa790aa5e1a75a"}, - {file = "aiohttp-3.14.3-cp312-cp312-win_arm64.whl", hash = "sha256:362a3fd481769cac1a824514bcd86fda51c65e8fe6e051099e008fddde6db17c"}, - {file = "aiohttp-3.14.3-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:2e9878ae68e4a5f1c0abe4dd497dbc3d51946f5837b56759e2a02e78fa90ef86"}, - {file = "aiohttp-3.14.3-cp313-cp313-android_21_x86_64.whl", hash = "sha256:f3d2669fe7dec7fc359ecdb5984b29b50d85d5d00f8c1cb61de4f4a24ee42627"}, - {file = "aiohttp-3.14.3-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:cc7cb243a68167172f48c1fd43cee91ec4b1d40cefd190edd43369d1a6bc9c82"}, - {file = "aiohttp-3.14.3-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:78253b573e6ffab5028924fc98bc281aae05445969982a10864bc360dea2016c"}, - {file = "aiohttp-3.14.3-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7041d52c3a7fa20c9e8c182b534704abb19502c8bdcbde7ab23bfda6f642394f"}, - {file = "aiohttp-3.14.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ac74facc01463f138b0da5580329cfcc82818dea5656e83ddcd11268fc12ff80"}, - {file = "aiohttp-3.14.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d6218d92e450824e9b4881f44e8c09f1853b490f9a64130801024a4793b1b3b0"}, - {file = "aiohttp-3.14.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:11fb37ef075669eee52ab1928fbf6e1741fada40409fa309ebde9607a962aebf"}, - {file = "aiohttp-3.14.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:55bdcc472aafe2de4a253045cc128007a64f1e0264fb675791e132ea5edaa3bd"}, - {file = "aiohttp-3.14.3-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c39846c3aad97a8530c89d7a3869a8f8e9e3762c6ac0504481e5c80948f7e807"}, - {file = "aiohttp-3.14.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5895ef58c4620afe02fa16044f023dc4dafec08158f9d08874a46a7dbc0341b8"}, - {file = "aiohttp-3.14.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa9467a8113aa69d3d7c55a70ef0b7c636010a40993f3df9d9d0d73b3eb7ef24"}, - {file = "aiohttp-3.14.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d7d2deec16eeedf55f2c7cf75b521ea3856a5177e123844f8fd0f114ce252cb5"}, - {file = "aiohttp-3.14.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:dd54d0e8717de95939766febac482ac0474d8ac3b048115f9f2b1d23a16e7db4"}, - {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:df82f3787c940c94986b34222d59c9e38843fba85139f36e85255a82ad5355a9"}, - {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:42a67efc36300d052fb4508a53e8b6901b9284b599ae63945c377569c5fcc1e1"}, - {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:7a75aa63cbf9b21cfaf60dc2657e19df2c2867d91707d653fee171ffeedd1371"}, - {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:e92eb8acc45eb6a9f4935071a77edf5b85cc6f8dfad5cd99e97653c26593cdde"}, - {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b014a6ed7cf912e787149fdc529166d3ceabac23f26efeea3158c9aba2354e7e"}, - {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3d4f72af88ac2474bb5bca640030320e3d38a0163a1d7533500e87be458eef71"}, - {file = "aiohttp-3.14.3-cp313-cp313-win32.whl", hash = "sha256:5f08ec777f35ee70720233b8b9811d3bb5d728137f30ac91b7457709c3261ac0"}, - {file = "aiohttp-3.14.3-cp313-cp313-win_amd64.whl", hash = "sha256:dff9461ec275f22135650d5ba4b4931a11f3958df7dfbb8db630000d4dee0883"}, - {file = "aiohttp-3.14.3-cp313-cp313-win_arm64.whl", hash = "sha256:ddcac3c6b382e81f1dd0499199d4136b877beb4cb5ef770bbbfba56c4b8f55d2"}, - {file = "aiohttp-3.14.3-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:49f7325beb0f85ef4aef5f48f490269575f83e6e2acad00a1d80b807eb027062"}, - {file = "aiohttp-3.14.3-cp314-cp314-android_24_x86_64.whl", hash = "sha256:e3be98a7c30b8c25d573dafba7171d66dfb05ee6a9070fc46535464ff97700a6"}, - {file = "aiohttp-3.14.3-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:614c61d478b83953e261d02bb2df750f17227cd33ef8002945bf5aebbde21919"}, - {file = "aiohttp-3.14.3-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:1caa7b0d05f3e3a36f87788c59e970a7ee1cefcfcbb924a9f138c4a6551c9cb7"}, - {file = "aiohttp-3.14.3-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:dfa68deb2a443bdaa3ea5297b0699c1464f08aef3812b486d1348eee61b07dc0"}, - {file = "aiohttp-3.14.3-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:e72ee89e28d907a18f46959b4eb0bb06701cc7f8cf4366e00029e2ccfaaf5924"}, - {file = "aiohttp-3.14.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ad4c8b7488d745d2ca4838ebd8ae5ba9b56341d30b1da43640e4ce87f9f49646"}, - {file = "aiohttp-3.14.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:db332af25642007330fca8be5c4d194caf2bea7a7fc84415aff3497af5dfee6b"}, - {file = "aiohttp-3.14.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:25bd2708db6bdf6a6630dd37bdcdfcb47c4434d22ac69c64665b802910140b30"}, - {file = "aiohttp-3.14.3-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:cef89a58e628c4efcac3275c2d68083f82426dcdc89c1492a6f654f9f7ea6ab9"}, - {file = "aiohttp-3.14.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c23ec8ee9d5ab2f5421f9c7fffce208435607af27fd46d4a44e031954352838f"}, - {file = "aiohttp-3.14.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e2667f0bbe7eb6c74eae5e9691441ad186e5845ca3cff63230fc09c4e7514f5d"}, - {file = "aiohttp-3.14.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:18cb43369747b2ae007bd2655fb8e63a099c2ff1d207962943636dac989b3147"}, - {file = "aiohttp-3.14.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d77640cc618c1d99fc4f8589c0f24a730adfa54eb1e57ef7bf0c8dfb78da898c"}, - {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:53e5179d8abb5710f8e83ba207c41c8d1261fcffd4616500e15ca2b7a33be10a"}, - {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:cd817772b2fcf2b8c0905795318485f9ec16eae60b29feb7f4c77085311637f0"}, - {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4e3ac92d90e92773b2362d506068e9a948192bd553e743c5b2429e28527c8661"}, - {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:3f42e9b78301f11c8f861746175d8b9c1ccef713fcad9eab396e2f6db8ed4a22"}, - {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:9d9edccfe496b476db5f398d97b865e9a6752bcf8aec4eef8390ce20fb64bb41"}, - {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:1c5ec8fb1bcc31a8466f74aaf26c345d5c386fa4bd08a3f0eb9c7a4a3fe8b5bf"}, - {file = "aiohttp-3.14.3-cp314-cp314-win32.whl", hash = "sha256:38901a84da3ce22249f6e860bf8f90d141bcab7da090cc398f8bb58c0e44b7da"}, - {file = "aiohttp-3.14.3-cp314-cp314-win_amd64.whl", hash = "sha256:8b3b60de05f3dcb6f6a00f818bb2ec781cee4de0645f59ccaf99b1d1823b6100"}, - {file = "aiohttp-3.14.3-cp314-cp314-win_arm64.whl", hash = "sha256:1576145bdceeb92382d899751e12743a3a5b8e460a841e3e50543859e54864dc"}, - {file = "aiohttp-3.14.3-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:8800c996b01c2772a783e3e46f3e1abd5823029adca0df54231960de9bfefa5b"}, - {file = "aiohttp-3.14.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ebe8e504f058fe91223351cecd2d9d6946c9d241bb0250d898ffbdf584cc72b0"}, - {file = "aiohttp-3.14.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:30402d03a7c0ff52bce290b57e564e9079fd9d0cb545c8aba73f86a103162d2e"}, - {file = "aiohttp-3.14.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9fc7b5bfec6573f3ae844f457fdde5adeb713f8b8e4a81ad64fc207b49383716"}, - {file = "aiohttp-3.14.3-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8a5fd34f7f7410d1730d5c2ba873cacb2eed3fede366feb268a70ba22581ed8f"}, - {file = "aiohttp-3.14.3-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:270d3dace9ca2f10f0da5d8ebe519b7a310fc6112ed916e32df5866df0888553"}, - {file = "aiohttp-3.14.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3ae5b3a59436d089b5395d910121a390feed4d00578eb95a0fd1a329fe963100"}, - {file = "aiohttp-3.14.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2498f0fe69ead802f9675beca44a7c21c62fdaa4ec5145ea1c3ad6edbee29f85"}, - {file = "aiohttp-3.14.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a0dc483c00da8b673abbb367eb6f8d8f4bcec30eb58529ea13cb42e7fd2dfa33"}, - {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c7d3a97c678d34fc5b59da671ee9cd630096ddc643e7b5a30d54a2a6f3574d3f"}, - {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:f8fb78a83c9e5f741ca3a68cfb455c1f5bb83b4e7249a3848b3cd78d0a8563b0"}, - {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:74ab5b6a9fb13e873e5a90946588baecaf488745e1db1a4a5c433f971f035098"}, - {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:bd52f811e65f6fb634b1047159657c98f52b407f8efec907bcfc09da9a4c0a25"}, - {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f0f177d1b195b9e06376cfd7d308d8a1b920909a609d03ac82a8c73bbb16d3b9"}, - {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:498c6c623134f8e09a3c4e60bcd607a0b4590dd7dbf08dd40851b27cbb520ccb"}, - {file = "aiohttp-3.14.3-cp314-cp314t-win32.whl", hash = "sha256:b304db572b4368edd8dda8a2274f73156fe15558fca4a917cb8a09fc47af5963"}, - {file = "aiohttp-3.14.3-cp314-cp314t-win_amd64.whl", hash = "sha256:b20032766aedf6261c7a566585a40867d092ac03a0d81592d5370ef9b054f99b"}, - {file = "aiohttp-3.14.3-cp314-cp314t-win_arm64.whl", hash = "sha256:2e1161602f45a54de2ce0905243a95f58cb42dcd378402f3697f5e0b21e9d2e7"}, - {file = "aiohttp-3.14.3.tar.gz", hash = "sha256:9491196535a88924a60afd5b5f434b5b203b6cc616250878dbdb223a8f7844bc"}, -] - -[package.dependencies] -aiohappyeyeballs = ">=2.5.0" -aiosignal = ">=1.4.0" -attrs = ">=17.3.0" -frozenlist = ">=1.1.1" -multidict = ">=4.5,<7.0" -propcache = ">=0.2.0" -typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} -yarl = ">=1.17.0,<2.0" - -[package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] - -[[package]] -name = "aiosignal" -version = "1.4.0" -description = "aiosignal: a list of registered asynchronous callbacks" -optional = false -python-versions = ">=3.9" -groups = ["dev"] -files = [ - {file = "aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e"}, - {file = "aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7"}, -] - -[package.dependencies] -frozenlist = ">=1.1.0" -typing-extensions = {version = ">=4.2", markers = "python_version < \"3.13\""} - [[package]] name = "annotated-types" version = "0.8.0" @@ -182,18 +12,6 @@ files = [ {file = "annotated_types-0.8.0.tar.gz", hash = "sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7"}, ] -[[package]] -name = "attrs" -version = "26.1.0" -description = "Classes Without Boilerplate" -optional = false -python-versions = ">=3.9" -groups = ["dev"] -files = [ - {file = "attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309"}, - {file = "attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32"}, -] - [[package]] name = "babel" version = "2.18.0" @@ -489,146 +307,6 @@ files = [ ] markers = {dev = "sys_platform == \"win32\""} -[[package]] -name = "frozenlist" -version = "1.8.0" -description = "A list-like structure which implements collections.abc.MutableSequence" -optional = false -python-versions = ">=3.9" -groups = ["dev"] -files = [ - {file = "frozenlist-1.8.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b37f6d31b3dcea7deb5e9696e529a6aa4a898adc33db82da12e4c60a7c4d2011"}, - {file = "frozenlist-1.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef2b7b394f208233e471abc541cc6991f907ffd47dc72584acee3147899d6565"}, - {file = "frozenlist-1.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a88f062f072d1589b7b46e951698950e7da00442fc1cacbe17e19e025dc327ad"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f57fb59d9f385710aa7060e89410aeb5058b99e62f4d16b08b91986b9a2140c2"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:799345ab092bee59f01a915620b5d014698547afd011e691a208637312db9186"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c23c3ff005322a6e16f71bf8692fcf4d5a304aaafe1e262c98c6d4adc7be863e"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8a76ea0f0b9dfa06f254ee06053d93a600865b3274358ca48a352ce4f0798450"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c7366fe1418a6133d5aa824ee53d406550110984de7637d65a178010f759c6ef"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:13d23a45c4cebade99340c4165bd90eeb4a56c6d8a9d8aa49568cac19a6d0dc4"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:e4a3408834f65da56c83528fb52ce7911484f0d1eaf7b761fc66001db1646eff"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:42145cd2748ca39f32801dad54aeea10039da6f86e303659db90db1c4b614c8c"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e2de870d16a7a53901e41b64ffdf26f2fbb8917b3e6ebf398098d72c5b20bd7f"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:20e63c9493d33ee48536600d1a5c95eefc870cd71e7ab037763d1fbb89cc51e7"}, - {file = "frozenlist-1.8.0-cp310-cp310-win32.whl", hash = "sha256:adbeebaebae3526afc3c96fad434367cafbfd1b25d72369a9e5858453b1bb71a"}, - {file = "frozenlist-1.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:667c3777ca571e5dbeb76f331562ff98b957431df140b54c85fd4d52eea8d8f6"}, - {file = "frozenlist-1.8.0-cp310-cp310-win_arm64.whl", hash = "sha256:80f85f0a7cc86e7a54c46d99c9e1318ff01f4687c172ede30fd52d19d1da1c8e"}, - {file = "frozenlist-1.8.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:09474e9831bc2b2199fad6da3c14c7b0fbdd377cce9d3d77131be28906cb7d84"}, - {file = "frozenlist-1.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:17c883ab0ab67200b5f964d2b9ed6b00971917d5d8a92df149dc2c9779208ee9"}, - {file = "frozenlist-1.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa47e444b8ba08fffd1c18e8cdb9a75db1b6a27f17507522834ad13ed5922b93"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2552f44204b744fba866e573be4c1f9048d6a324dfe14475103fd51613eb1d1f"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:957e7c38f250991e48a9a73e6423db1bb9dd14e722a10f6b8bb8e16a0f55f695"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8585e3bb2cdea02fc88ffa245069c36555557ad3609e83be0ec71f54fd4abb52"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:edee74874ce20a373d62dc28b0b18b93f645633c2943fd90ee9d898550770581"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c9a63152fe95756b85f31186bddf42e4c02c6321207fd6601a1c89ebac4fe567"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b6db2185db9be0a04fecf2f241c70b63b1a242e2805be291855078f2b404dd6b"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:f4be2e3d8bc8aabd566f8d5b8ba7ecc09249d74ba3c9ed52e54dc23a293f0b92"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:c8d1634419f39ea6f5c427ea2f90ca85126b54b50837f31497f3bf38266e853d"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:1a7fa382a4a223773ed64242dbe1c9c326ec09457e6b8428efb4118c685c3dfd"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:11847b53d722050808926e785df837353bd4d75f1d494377e59b23594d834967"}, - {file = "frozenlist-1.8.0-cp311-cp311-win32.whl", hash = "sha256:27c6e8077956cf73eadd514be8fb04d77fc946a7fe9f7fe167648b0b9085cc25"}, - {file = "frozenlist-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:ac913f8403b36a2c8610bbfd25b8013488533e71e62b4b4adce9c86c8cea905b"}, - {file = "frozenlist-1.8.0-cp311-cp311-win_arm64.whl", hash = "sha256:d4d3214a0f8394edfa3e303136d0575eece0745ff2b47bd2cb2e66dd92d4351a"}, - {file = "frozenlist-1.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1"}, - {file = "frozenlist-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b"}, - {file = "frozenlist-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa"}, - {file = "frozenlist-1.8.0-cp312-cp312-win32.whl", hash = "sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf"}, - {file = "frozenlist-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746"}, - {file = "frozenlist-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd"}, - {file = "frozenlist-1.8.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8d92f1a84bb12d9e56f818b3a746f3efba93c1b63c8387a73dde655e1e42282a"}, - {file = "frozenlist-1.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:96153e77a591c8adc2ee805756c61f59fef4cf4073a9275ee86fe8cba41241f7"}, - {file = "frozenlist-1.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f21f00a91358803399890ab167098c131ec2ddd5f8f5fd5fe9c9f2c6fcd91e40"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eaa352d7047a31d87dafcacbabe89df0aa506abb5b1b85a2fb91bc3faa02d822"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:03ae967b4e297f58f8c774c7eabcce57fe3c2434817d4385c50661845a058121"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f6292f1de555ffcc675941d65fffffb0a5bcd992905015f85d0592201793e0e5"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29548f9b5b5e3460ce7378144c3010363d8035cea44bc0bf02d57f5a685e084e"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ec3cc8c5d4084591b4237c0a272cc4f50a5b03396a47d9caaf76f5d7b38a4f11"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:517279f58009d0b1f2e7c1b130b377a349405da3f7621ed6bfae50b10adf20c1"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:db1e72ede2d0d7ccb213f218df6a078a9c09a7de257c2fe8fcef16d5925230b1"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b4dec9482a65c54a5044486847b8a66bf10c9cb4926d42927ec4e8fd5db7fed8"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:21900c48ae04d13d416f0e1e0c4d81f7931f73a9dfa0b7a8746fb2fe7dd970ed"}, - {file = "frozenlist-1.8.0-cp313-cp313-win32.whl", hash = "sha256:8b7b94a067d1c504ee0b16def57ad5738701e4ba10cec90529f13fa03c833496"}, - {file = "frozenlist-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:878be833caa6a3821caf85eb39c5ba92d28e85df26d57afb06b35b2efd937231"}, - {file = "frozenlist-1.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:44389d135b3ff43ba8cc89ff7f51f5a0bb6b63d829c8300f79a2fe4fe61bcc62"}, - {file = "frozenlist-1.8.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:e25ac20a2ef37e91c1b39938b591457666a0fa835c7783c3a8f33ea42870db94"}, - {file = "frozenlist-1.8.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:07cdca25a91a4386d2e76ad992916a85038a9b97561bf7a3fd12d5d9ce31870c"}, - {file = "frozenlist-1.8.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4e0c11f2cc6717e0a741f84a527c52616140741cd812a50422f83dc31749fb52"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b3210649ee28062ea6099cfda39e147fa1bc039583c8ee4481cb7811e2448c51"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:581ef5194c48035a7de2aefc72ac6539823bb71508189e5de01d60c9dcd5fa65"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3ef2d026f16a2b1866e1d86fc4e1291e1ed8a387b2c333809419a2f8b3a77b82"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5500ef82073f599ac84d888e3a8c1f77ac831183244bfd7f11eaa0289fb30714"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:50066c3997d0091c411a66e710f4e11752251e6d2d73d70d8d5d4c76442a199d"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:5c1c8e78426e59b3f8005e9b19f6ff46e5845895adbde20ece9218319eca6506"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:eefdba20de0d938cec6a89bd4d70f346a03108a19b9df4248d3cf0d88f1b0f51"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:cf253e0e1c3ceb4aaff6df637ce033ff6535fb8c70a764a8f46aafd3d6ab798e"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6da155091429aeba16851ecb10a9104a108bcd32f6c1642867eadaee401c1c41"}, - {file = "frozenlist-1.8.0-cp313-cp313t-win32.whl", hash = "sha256:0f96534f8bfebc1a394209427d0f8a63d343c9779cda6fc25e8e121b5fd8555b"}, - {file = "frozenlist-1.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:5d63a068f978fc69421fb0e6eb91a9603187527c86b7cd3f534a5b77a592b888"}, - {file = "frozenlist-1.8.0-cp313-cp313t-win_arm64.whl", hash = "sha256:bf0a7e10b077bf5fb9380ad3ae8ce20ef919a6ad93b4552896419ac7e1d8e042"}, - {file = "frozenlist-1.8.0-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:cee686f1f4cadeb2136007ddedd0aaf928ab95216e7691c63e50a8ec066336d0"}, - {file = "frozenlist-1.8.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:119fb2a1bd47307e899c2fac7f28e85b9a543864df47aa7ec9d3c1b4545f096f"}, - {file = "frozenlist-1.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4970ece02dbc8c3a92fcc5228e36a3e933a01a999f7094ff7c23fbd2beeaa67c"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:cba69cb73723c3f329622e34bdbf5ce1f80c21c290ff04256cff1cd3c2036ed2"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:778a11b15673f6f1df23d9586f83c4846c471a8af693a22e066508b77d201ec8"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:97260ff46b207a82a7567b581ab4190bd4dfa09f4db8a8b49d1a958f6aa4940e"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:54b2077180eb7f83dd52c40b2750d0a9f175e06a42e3213ce047219de902717a"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2f05983daecab868a31e1da44462873306d3cbfd76d1f0b5b69c473d21dbb128"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:33f48f51a446114bc5d251fb2954ab0164d5be02ad3382abcbfe07e2531d650f"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:154e55ec0655291b5dd1b8731c637ecdb50975a2ae70c606d100750a540082f7"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:4314debad13beb564b708b4a496020e5306c7333fa9a3ab90374169a20ffab30"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:073f8bf8becba60aa931eb3bc420b217bb7d5b8f4750e6f8b3be7f3da85d38b7"}, - {file = "frozenlist-1.8.0-cp314-cp314-win32.whl", hash = "sha256:bac9c42ba2ac65ddc115d930c78d24ab8d4f465fd3fc473cdedfccadb9429806"}, - {file = "frozenlist-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:3e0761f4d1a44f1d1a47996511752cf3dcec5bbdd9cc2b4fe595caf97754b7a0"}, - {file = "frozenlist-1.8.0-cp314-cp314-win_arm64.whl", hash = "sha256:d1eaff1d00c7751b7c6662e9c5ba6eb2c17a2306ba5e2a37f24ddf3cc953402b"}, - {file = "frozenlist-1.8.0-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:d3bb933317c52d7ea5004a1c442eef86f426886fba134ef8cf4226ea6ee1821d"}, - {file = "frozenlist-1.8.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:8009897cdef112072f93a0efdce29cd819e717fd2f649ee3016efd3cd885a7ed"}, - {file = "frozenlist-1.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2c5dcbbc55383e5883246d11fd179782a9d07a986c40f49abe89ddf865913930"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:39ecbc32f1390387d2aa4f5a995e465e9e2f79ba3adcac92d68e3e0afae6657c"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92db2bf818d5cc8d9c1f1fc56b897662e24ea5adb36ad1f1d82875bd64e03c24"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2dc43a022e555de94c3b68a4ef0b11c4f747d12c024a520c7101709a2144fb37"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cb89a7f2de3602cfed448095bab3f178399646ab7c61454315089787df07733a"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:33139dc858c580ea50e7e60a1b0ea003efa1fd42e6ec7fdbad78fff65fad2fd2"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:168c0969a329b416119507ba30b9ea13688fafffac1b7822802537569a1cb0ef"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:28bd570e8e189d7f7b001966435f9dac6718324b5be2990ac496cf1ea9ddb7fe"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b2a095d45c5d46e5e79ba1e5b9cb787f541a8dee0433836cea4b96a2c439dcd8"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:eab8145831a0d56ec9c4139b6c3e594c7a83c2c8be25d5bcf2d86136a532287a"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:974b28cf63cc99dfb2188d8d222bc6843656188164848c4f679e63dae4b0708e"}, - {file = "frozenlist-1.8.0-cp314-cp314t-win32.whl", hash = "sha256:342c97bf697ac5480c0a7ec73cd700ecfa5a8a40ac923bd035484616efecc2df"}, - {file = "frozenlist-1.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:06be8f67f39c8b1dc671f5d83aaefd3358ae5cdcf8314552c57e7ed3e6475bdd"}, - {file = "frozenlist-1.8.0-cp314-cp314t-win_arm64.whl", hash = "sha256:102e6314ca4da683dca92e3b1355490fed5f313b768500084fbe6371fddfdb79"}, - {file = "frozenlist-1.8.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d8b7138e5cd0647e4523d6685b0eac5d4be9a184ae9634492f25c6eb38c12a47"}, - {file = "frozenlist-1.8.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a6483e309ca809f1efd154b4d37dc6d9f61037d6c6a81c2dc7a15cb22c8c5dca"}, - {file = "frozenlist-1.8.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1b9290cf81e95e93fdf90548ce9d3c1211cf574b8e3f4b3b7cb0537cf2227068"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:59a6a5876ca59d1b63af8cd5e7ffffb024c3dc1e9cf9301b21a2e76286505c95"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6dc4126390929823e2d2d9dc79ab4046ed74680360fc5f38b585c12c66cdf459"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:332db6b2563333c5671fecacd085141b5800cb866be16d5e3eb15a2086476675"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9ff15928d62a0b80bb875655c39bf517938c7d589554cbd2669be42d97c2cb61"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7bf6cdf8e07c8151fba6fe85735441240ec7f619f935a5205953d58009aef8c6"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:48e6d3f4ec5c7273dfe83ff27c91083c6c9065af655dc2684d2c200c94308bb5"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:1a7607e17ad33361677adcd1443edf6f5da0ce5e5377b798fba20fae194825f3"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:5a3a935c3a4e89c733303a2d5a7c257ea44af3a56c8202df486b7f5de40f37e1"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:940d4a017dbfed9daf46a3b086e1d2167e7012ee297fef9e1c545c4d022f5178"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:b9be22a69a014bc47e78072d0ecae716f5eb56c15238acca0f43d6eb8e4a5bda"}, - {file = "frozenlist-1.8.0-cp39-cp39-win32.whl", hash = "sha256:1aa77cb5697069af47472e39612976ed05343ff2e84a3dcf15437b232cbfd087"}, - {file = "frozenlist-1.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:7398c222d1d405e796970320036b1b563892b65809d9e5261487bb2c7f7b5c6a"}, - {file = "frozenlist-1.8.0-cp39-cp39-win_arm64.whl", hash = "sha256:b4f3b365f31c6cd4af24545ca0a244a53688cad8834e32f56831c4923b50a103"}, - {file = "frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d"}, - {file = "frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad"}, -] - [[package]] name = "ghp-import" version = "2.1.0" @@ -653,7 +331,7 @@ version = "3.19" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" -groups = ["main", "dev"] +groups = ["main"] files = [ {file = "idna-3.19-py3-none-any.whl", hash = "sha256:815e7be7a7806d54abb586dc943addc79e8b2ee16915059658cbeff4b1b43bf4"}, {file = "idna-3.19.tar.gz", hash = "sha256:5e0811a4383b21dc5838069f801c4fb62113b7447663d2530d2bd6e77b49bf15"}, @@ -692,25 +370,6 @@ MarkupSafe = ">=2.0" [package.extras] i18n = ["Babel (>=2.7)"] -[[package]] -name = "linkcheckmd" -version = "1.4.0" -description = "Check links for Markdown-based site" -optional = false -python-versions = ">=3.7" -groups = ["dev"] -files = [ - {file = "linkcheckmd-1.4.0.tar.gz", hash = "sha256:3a539c9a4e11697fc7fcc269d379accf93c8cccbf971f3cea0bae40912d9f609"}, -] - -[package.dependencies] -aiohttp = ">=3.3" - -[package.extras] -lint = ["flake8", "flake8-blind-except", "flake8-bugbear", "flake8-builtins", "mypy"] -requests = ["requests"] -tests = ["pytest"] - [[package]] name = "markdown" version = "3.10.3" @@ -953,187 +612,6 @@ files = [ {file = "mkdocs_material_extensions-1.3.1.tar.gz", hash = "sha256:10c9511cea88f568257f960358a467d12b970e1f7b2c0e5fb2bb48cab1928443"}, ] -[[package]] -name = "multidict" -version = "6.8.0" -description = "multidict implementation" -optional = false -python-versions = ">=3.10" -groups = ["dev"] -files = [ - {file = "multidict-6.8.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c0fe437a6d2f36aac2b49517057776575b5bf359df314cca20d230a6e139c089"}, - {file = "multidict-6.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f3a0a31189acf6703307397c6139ddabd734c20c5ef92649fc93e473df6615a3"}, - {file = "multidict-6.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c9c98d2f0126ba84cb45601eed97ff67ff767e19ae6eb3c31b02827b54d700e5"}, - {file = "multidict-6.8.0-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:7bc7003991ebd368a20d05228137a37b3d3066751f3ea1e4f7b8efe8e752f2f5"}, - {file = "multidict-6.8.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c2c5fd0fd39574ccd58e1a52565b341aff522c5c836f1b3eb7605c371e61f52c"}, - {file = "multidict-6.8.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b1cd4d66ce894a45482e1ac2837c31d0bd447df35065e542b60055aa2d00404b"}, - {file = "multidict-6.8.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7a2573d0fd34f361a4a14e54d8cda3a91ac4e55fbf0d719698024f3b09c5b147"}, - {file = "multidict-6.8.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:edda19aff836ec515caafc09ea53d2ab144a041f09ee9a7cefcbd3ae4e976256"}, - {file = "multidict-6.8.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e418ec99574ca24365ca96546af285c2b021a1a072478a79f0e3cc3b08837154"}, - {file = "multidict-6.8.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:11d71490bf4bbff1141b14b93af419ad68c56b60bea9277fcb3f94dcca4796eb"}, - {file = "multidict-6.8.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e8e1e895e23818d343e4ae7dd95a0a556fdeaf8b471acf1c0a39b93c6f54d478"}, - {file = "multidict-6.8.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:15db8e6cab5f4cc9241bc56e69fdf3452cf49c10ee3c7977c742e68a275b3786"}, - {file = "multidict-6.8.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:8dc2d9c3a924ed14166e63650b2cf9f59e7821743bdd50b23802bd97ca09bde5"}, - {file = "multidict-6.8.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:f2fa3d3b1c933d4bcb8fd2018700d5e7235c52f2ab8c88d22286965c5c0f00f8"}, - {file = "multidict-6.8.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:d6dc7804c50fabd28644d4d18a4b20aad3681b3e64f3acd3182b330ca73f7a32"}, - {file = "multidict-6.8.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:0eca15d627e942ce186a935061f1568cc46c02e97c419c8da802df2be9f917d8"}, - {file = "multidict-6.8.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d7e5ba0a0153e35fbce9c51df530c8b4cb0c3012b46a04ff9a048441a269c2ed"}, - {file = "multidict-6.8.0-cp310-cp310-win32.whl", hash = "sha256:b7e62b8fc7bd6cad007b9f2e0ad9c8d4854c06350d5f51e1a439dd18b510ecac"}, - {file = "multidict-6.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:dc911ae6152e455b16a2a1a626aa6cd612fa01efb9d0a4ab3f5cf328b911483d"}, - {file = "multidict-6.8.0-cp310-cp310-win_arm64.whl", hash = "sha256:6300d5176647145ba1e22991c924fb29743e54b4d7b8bc85a0d3ec0e55e189cb"}, - {file = "multidict-6.8.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:1b8986d4313dcee7c932837d16a535f1840b827bac1ea7c5c4c80751d0423794"}, - {file = "multidict-6.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2622fe114c0bd66ca5c461859357587f5a5e35ee5ff49fc5643d1bc78dbb41c6"}, - {file = "multidict-6.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:26a7aafc992e78872e2c8c1f7248c0e01139cf9020a7781b0c064fa566832712"}, - {file = "multidict-6.8.0-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:05c2e90c5289c5f7436ba2c25812a5fbdaa1c1bc11c8d8d3bbf64f5cd7c633dd"}, - {file = "multidict-6.8.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7941ef106ca1f2c62314a13c7ed913bcf49641f3efdc12864d588e17870920ac"}, - {file = "multidict-6.8.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a5a7ee1217949ddd43c6b7bcf70d5c22193bb50e8c695386de5905325e93ce9f"}, - {file = "multidict-6.8.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6b62b7e0025aa48dec11e125e655d1157985a5fdcec04b1ad500101ad072b891"}, - {file = "multidict-6.8.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ea880d441be7c510106bc56064be39266d948aef94ad4955e8784690019a5d9f"}, - {file = "multidict-6.8.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ab83fdd8cf307353edba9c427c17a3a021c2522d690f5633dd9f72d28b48ccca"}, - {file = "multidict-6.8.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3d1f48582686a0a3b81e9b43234766cc96697df72081af3f48107bd3f34d34e5"}, - {file = "multidict-6.8.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:563661919f603374c40cf45ffcd25535c12b8954203569a2ab1cee5265871cf4"}, - {file = "multidict-6.8.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:2ba9933e8f35fe4a70f540b837254c4055da82dc3a9e500a8f95e61498083a15"}, - {file = "multidict-6.8.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:68d40b2bace413f3231f5729d3fcfb1837fd31c4907e241b5d43211bfd76f3c2"}, - {file = "multidict-6.8.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a9e246f67ac038568b854ed7c5578e4c6af1f742359901a8fcc3603ff1358df6"}, - {file = "multidict-6.8.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:29b6e7bc4442a56cf8e0dc1cabf3fdc77cd533568d6829fc76a1effd2ce332ec"}, - {file = "multidict-6.8.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:82780eb8bf59e8fb25dd081fde6e058805045d6374a7f2f877effc826ca4434b"}, - {file = "multidict-6.8.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:2196ba6df392c3574acadd14ef87550f3611349c8618564de324b806a7a31cee"}, - {file = "multidict-6.8.0-cp311-cp311-win32.whl", hash = "sha256:b8b7aa75146266fd3e2a2437cf69ae188688c04ab8665b163d4257b46c1e0c83"}, - {file = "multidict-6.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:b03ca066b47b18b205cc080dca6f76cbd159f8cdd33a02a0700164c13b37e463"}, - {file = "multidict-6.8.0-cp311-cp311-win_arm64.whl", hash = "sha256:54af1266710cb0f305127ae0b970aff8d208057f8a29cd6e1db99b0114947035"}, - {file = "multidict-6.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:22a310ad37672a261e55a8b5e28d0ae08cfb68abb1f46418ccd19835c3b8e836"}, - {file = "multidict-6.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bb8c7da8c861391f7ae48e3593762be2dabe405109e01aec520fbe1a6d15d14b"}, - {file = "multidict-6.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:10456943903744ae1249728161c96bd9d2f7eb5ee17fcc2ffda2dc32e1bb36c7"}, - {file = "multidict-6.8.0-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:658f5a1895b804423d97b22d06fc0d0b171c7c01dcc3aa9c8faf0c0e26a249a5"}, - {file = "multidict-6.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90c10b22860dbd09982d0b8993b66231a861bea2993d4a817ff35273f6ea285a"}, - {file = "multidict-6.8.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:088b04a66b3c1fce6fe4d771ec184a0426262d0b86709c908477b4ac7965df40"}, - {file = "multidict-6.8.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9caef53b20a105c0d66518a34be2f71b2783de8d091767575ef86f6ea422236d"}, - {file = "multidict-6.8.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a5e1583c14775580da05641240ce0d93f36ce3ddef3d5083a827468b0bcfe874"}, - {file = "multidict-6.8.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:003a3bddb32915c3f67096ea41d24e53edf710edb65a1f5d0c70ab40b0e4d20b"}, - {file = "multidict-6.8.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:35977263d9bf506dbc65349f63b3b8c91606d4abc110990945e3b94bc671319c"}, - {file = "multidict-6.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e9dc7b4ff6ef184504b49ef9a4113d49a646653b2ce89f5f48c1f57cdf6ba081"}, - {file = "multidict-6.8.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:887f9a975996032c686719eb7b3e1e7942fab5079c2b778bbd9afe9a9d78244f"}, - {file = "multidict-6.8.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:f3071e6515cc63714d014da8f738ae9fa3997c476203f3cd46de380c2376ed7b"}, - {file = "multidict-6.8.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c5f3a2af441670d80ce5fdf13b6c1b421fc1fc7fc5182d58ac7486738bb2b742"}, - {file = "multidict-6.8.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:31e8901637e20ccb3cf8f8848b5d0f7a00462bf5b34f7cf3dcbb2753b18e8b39"}, - {file = "multidict-6.8.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:13967dca8b2f33230a1427b52438326bb1c9101a1df22a3309ed3fcbbb3c96f0"}, - {file = "multidict-6.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ad474c11d851b6fc97cb625e4822bc0cbd567fc07dc2602e28faec5a36b42bbb"}, - {file = "multidict-6.8.0-cp312-cp312-win32.whl", hash = "sha256:7bb0dad75068fee80fcb60f88569722c199d8656a16706702dc6e3b786819c90"}, - {file = "multidict-6.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:7d26dc8f070c0ec5579e987fa615ffd6883086106eefdff9e10d160fc5630630"}, - {file = "multidict-6.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:e6ec7d37841609a691b96a10b4fde386c7cd93ebbb939f59c9f23325ee788395"}, - {file = "multidict-6.8.0-cp313-cp313-android_24_x86_64.whl", hash = "sha256:ec0a4d066356054d569a66e0a94691a2058b680be5e710298f61db11a3c4609f"}, - {file = "multidict-6.8.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:714597cb5d5e15a8a449d2ae23c45b486a9e8fa33c462c7a33d7f35b65d92943"}, - {file = "multidict-6.8.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:e0db3a4d1e264e225037a6023888972c25206a96e016021a5bea41c9a939f2a9"}, - {file = "multidict-6.8.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:27747162712e85c84598d364425dbf1714ff335bdb6ba3171c4e5081196e8916"}, - {file = "multidict-6.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:847d6082ae694dc95e548acb201bc100e1cfa96513bc71fdcb86f709dad6c435"}, - {file = "multidict-6.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:88a6df88567680504ae28bfa7a1f2f64243d91e79a40b2c92ef42efc531e23da"}, - {file = "multidict-6.8.0-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:560b211fc3bd4a1e1c6de44f6d38113bf5b410dfc89a4c0d2a3c0edbf1a0dfb8"}, - {file = "multidict-6.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:202436df907c15adbb94360296c425ea53cf8968a5d2cff9b5b9790ae1972b33"}, - {file = "multidict-6.8.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c46a08bf070d6849fed483e9d9833f9d06aecb8382ed985be0b38508b3ae958e"}, - {file = "multidict-6.8.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2cd560498ae8e1bcc955643c1d78eb8e338226d07a983c656ea8c4443d3eec0f"}, - {file = "multidict-6.8.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:758233648ac47b07c575224c4eadd73c8929c3b4c31e2afcfea935fde1cda735"}, - {file = "multidict-6.8.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:122adc7c46ac1e31ecfc7f81b2530533dccafdba70f5d741649f87e336c63384"}, - {file = "multidict-6.8.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8125e60f3c70e323ac07dd8b3635f7b3bbc5c3a9ac04ae5988f668ff7ae28a18"}, - {file = "multidict-6.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:83ff054b04915be5c15680da6c6012474a2cc2bf534129a0e8c6a99f17ba7238"}, - {file = "multidict-6.8.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:930c6058047410e3edff445f5a6e4457f2e089042dede00e2d18ce06f3ceae2e"}, - {file = "multidict-6.8.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:13e26f59f0eecfc5f67c663ad550ffdaf62c0f657547cde387f6c86af1c9449e"}, - {file = "multidict-6.8.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:fd789a294d8e098528be29b2669b83005ce569339f8cef167fc0274c3115c34c"}, - {file = "multidict-6.8.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:3126f2a96704505aa4e92a72d6e8a5d7f29d40a987ced8bf69e29d71dfc71fbc"}, - {file = "multidict-6.8.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:23c9ee89967b6a9b4048acb3b93b660ed714ce9c8bf3bbe652959bc120dc02dc"}, - {file = "multidict-6.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7a62e302fc8cd6aa8972207e7e951d1fdee7c1dda18568305041d19f0e2c00f5"}, - {file = "multidict-6.8.0-cp313-cp313-win32.whl", hash = "sha256:093167d22a8c95af30f597b8a5686f20a14512989942d4be804d119899caca20"}, - {file = "multidict-6.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:f25b61a708bd276e8cbb6afcbbf1b8e793a3be70ba0a842d0b8692020f83b706"}, - {file = "multidict-6.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:bb36381e1f9f9d06eba2f10bdd438e5d20c07d5b55e1a3eee30b9f44cbf52316"}, - {file = "multidict-6.8.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:f8b09b25e0f4dc2ea9e2adbb1cc3ba11a94d6fa3dd978ae659c8743052e1afbc"}, - {file = "multidict-6.8.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:1f57c414be82490bc0e0305fdb834186229b2d9b6a35fa0afd1eb1a772d125ab"}, - {file = "multidict-6.8.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:00be37bde741bf60871082cd347a093218c44886e99231b7516671c70f2c280d"}, - {file = "multidict-6.8.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:e37b744849fb631bb52e3dadde35ffeee365a6c41cf71257b5b7acc9cd83fd38"}, - {file = "multidict-6.8.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:c2b2a96cf1dd99fe7867be4c013314225f4d5786e6685906e29932d42aca6f11"}, - {file = "multidict-6.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:bea7df027015856ba5d0a88e3b4777ff8cb5c66b58fc108050fe79d4dd9d4d2d"}, - {file = "multidict-6.8.0-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:d3da668e903c934ed0b587ecacfed6901f6ae6384a6e975887592b61845e78bc"}, - {file = "multidict-6.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64eaeda36ee8d88f9e8616a587a8c66a663283cf6e0dcf013c1ddd8c758e4aef"}, - {file = "multidict-6.8.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ac746cb365bac1c462da9e3e6ab8904a8efe2217a56b0b2e3d9480f41d2b2602"}, - {file = "multidict-6.8.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:18f0e06360c3e451a3ab800355773c8d125a758238d780c800b0ee5e90ee903c"}, - {file = "multidict-6.8.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:69708fecaa88bcb2341397b49fc95057a835b02a3670c551b37f95dd79e64e3a"}, - {file = "multidict-6.8.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9606f583e7acaf61e7b3f56074e14037b9af7cb194590edfc0114b3ae5931ff7"}, - {file = "multidict-6.8.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1f66fe6a021173d0d47968491791966b9f3e6d61115f2491744aa0c07a6e67af"}, - {file = "multidict-6.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:be007d1aee2cbd530347dcafedb400891a3b5f1bd7135f95cf5d5b330b5219ee"}, - {file = "multidict-6.8.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:8457aff3c12a89a8e1c4674de5c777857fbc429f40fe117a3d29538547cbc364"}, - {file = "multidict-6.8.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:95c27b4f3f04320fc44e338573f40c5c956b504a7fcf081a157fd0b02579311c"}, - {file = "multidict-6.8.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:d244cf6b52b5ba1c34c3832f4652a668ebb36d95949b96eed9a1c54d916a90dd"}, - {file = "multidict-6.8.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5bbbb696c8024475b1877d14ce20d5f1cc05b8f6d786cea0fe3aa7fedc02e891"}, - {file = "multidict-6.8.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:cbd86f9787c5e2f5fd27d8b21458222f107347c6731c4e93dde68f554b466a2d"}, - {file = "multidict-6.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2f8a4b0b4d639d525928c7f30de527bfdf9ead6e44a5e8cb9c50aced5e4590cb"}, - {file = "multidict-6.8.0-cp314-cp314-win32.whl", hash = "sha256:8890c89d662560e51c55ac1304d6f919b23942abe9ae1127cb1de9aa6132fa52"}, - {file = "multidict-6.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:45cc39ba50fb0754a4359b90f8229ae08598fe2266abe3521b4e5a9ba916534a"}, - {file = "multidict-6.8.0-cp314-cp314-win_arm64.whl", hash = "sha256:d0264f8d5cb0a803f650a6a8572dfa0cd1e099a2234c588dc8fb220b415b865f"}, - {file = "multidict-6.8.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:1969971900b0871530f9b62280dcc2d75688e74d2a69262bc01faf2b96c78f04"}, - {file = "multidict-6.8.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:8180b635290a75af8478f1b3e9810135381ae24833293fe77b85c1c21ff842ab"}, - {file = "multidict-6.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:4261863fc8b5ab1b815ede94e592e94c6af5b04616014929057e61859e7382a9"}, - {file = "multidict-6.8.0-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:0b143d53590e89f43153d81d505a8448d4d57354354385aef8a51d67ffefa27e"}, - {file = "multidict-6.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:da1c112c5784ccd9d32cd90be6739fee32644e874eff6ae8f0497cba3e352e58"}, - {file = "multidict-6.8.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f7eefd0233a7c33ca980a5cfef26f1e9b5e2137839e752a99963696729f12d91"}, - {file = "multidict-6.8.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:348bb85e2038b40c007383616d73f734869063772372519549ebd7da1723d1a4"}, - {file = "multidict-6.8.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:095f62ea4e7a3be2f6c567ab695ce10e950f2adb905c1bec82281593e0b2d2ad"}, - {file = "multidict-6.8.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:be569fff1d85cd29391c431c5641c8772acb75bbdc61e60a8e82fceb9023d385"}, - {file = "multidict-6.8.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3533a03e4e789baf6a286e7b0b1b6da3f3d7c3eab569686ee29ee1d8b52e2cb4"}, - {file = "multidict-6.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:1bdb9b8fba5a9aef673ec90db3f55b1ce743f2fbdea4d37dc04d14ccdfc153ff"}, - {file = "multidict-6.8.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:f8d7b66c9e09c0bb0add2b5895e646b62a0849e71155066f215523de6b95cbe6"}, - {file = "multidict-6.8.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:563d6500ca80dac7bba6f48a78e0ffd87e21a7d4d24642c6503a2ddccd70c110"}, - {file = "multidict-6.8.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:346ac52e56bcda320c0dcdfdd081947ed7cada33afea4e2284bef7b0733bff9b"}, - {file = "multidict-6.8.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4ee953a5ebaeed38dc21cc032ed17a9d9782802e00042200497ab4b01b0bf7c0"}, - {file = "multidict-6.8.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:2f79cc3e8039a8cf5c77e0811b0807953fd52d0863b9b76970b20d696dc64a78"}, - {file = "multidict-6.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:43a4b56555bbcf8af161e7c7682bd93eec10f068c95844511864c018c8e5e13b"}, - {file = "multidict-6.8.0-cp314-cp314t-win32.whl", hash = "sha256:48ea524a25a1cd5972cf293bc95713918cba0bcd6fa9b992d906c857c546abe2"}, - {file = "multidict-6.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d0be2b832435001bc623ca7f1499ca1a853d4f082fb61221a80ce71132f50b26"}, - {file = "multidict-6.8.0-cp314-cp314t-win_arm64.whl", hash = "sha256:62b8e291a4f7edbf7cde7a43d831d893ba443a1b627498b53581943b0e348feb"}, - {file = "multidict-6.8.0-cp315-cp315-android_24_x86_64.whl", hash = "sha256:e192018b732f7b168e6604cbdf40fa8e05c996693b9eb445a0d8a73f4b77c5d3"}, - {file = "multidict-6.8.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:b25426f9f6ed402835617c8f23609a47045f91ecff365eb6734817e039a8ed25"}, - {file = "multidict-6.8.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:fa6c2880709c84457de104385b704fc28860f27e442ad13966fc4af8e714fe9c"}, - {file = "multidict-6.8.0-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:eabb03dc3e4ed6333ecd1cc9826ec80e7a98b5506deeb832d7260c8e44166d23"}, - {file = "multidict-6.8.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:59e539c4eb4d3a53b0e630a6ba2b2f2824732b5e73f90e30a280f12fde157b15"}, - {file = "multidict-6.8.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:835d5a90b11d1f5f8200ff3cc8316bded76eebebc92436398947a27657e645e7"}, - {file = "multidict-6.8.0-cp315-cp315-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:d2d236b8a44ae91536a12ebcb996bdb31cf27425f36b4d05c87f2ba2716050ba"}, - {file = "multidict-6.8.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bb9a60b7faa5d37c426fa91cf4d6738182a1f2755b9fab7c9c64cd466c4ce51e"}, - {file = "multidict-6.8.0-cp315-cp315-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0ef606c15cac6c90279acf34120784b6f36662cbf382defd3955cd8f1115336b"}, - {file = "multidict-6.8.0-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:68186a2d4051c8ffd17be33553bea2ec9bbc8ef860fe2980a221d96126296f31"}, - {file = "multidict-6.8.0-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2cc66abb85e2108c9ff8a1c0d20fa260bf690bbb33caef4ff3ecb2c2cbdfff5d"}, - {file = "multidict-6.8.0-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:69b3e519a132bb943b0daae15fc8c2168706b17f826481d32a32a5e784b129e3"}, - {file = "multidict-6.8.0-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e41226ecf607f062fe34a2f4cf64ad3a89e3a0180dc800b463b6b14c06dd10dc"}, - {file = "multidict-6.8.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:610c7637bc36b90f39e6c66f710f93d57018f83d53e1e187caaa218c6892b95f"}, - {file = "multidict-6.8.0-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:65c85c79f5a2c04fbbc18f006c014674dc5fdf270cb978d8862c82c6f694e60c"}, - {file = "multidict-6.8.0-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:628ff11e6720f90acd0c305dfa3339f04a783a20de8cda6ac333ba46447261e8"}, - {file = "multidict-6.8.0-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:0935971bffd0b479fc90c4811ca787703e93fcb6afea939a375dfc80285ab368"}, - {file = "multidict-6.8.0-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:9442b14eec262a1f74369bbd07e75bc5155105164649a4b9fbc1ebc7b8fb0b14"}, - {file = "multidict-6.8.0-cp315-cp315-musllinux_1_2_s390x.whl", hash = "sha256:397599503b718f0137f26d3f6532d6955069cd2e5917c47ef581495bc2529ff8"}, - {file = "multidict-6.8.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:9e37024b41d7a7e7e9cce14b248d54707c21c2a2ea30a47b71bdcefcafec00f2"}, - {file = "multidict-6.8.0-cp315-cp315-win32.whl", hash = "sha256:071da134651b04a8507dfb331ac0988f376337c2aea59486bf20989fb5b5a64e"}, - {file = "multidict-6.8.0-cp315-cp315-win_amd64.whl", hash = "sha256:3bafff8598f0528017ddc74194e5451d5c22d046c98935f8f86247b0f286e4f8"}, - {file = "multidict-6.8.0-cp315-cp315-win_arm64.whl", hash = "sha256:e886ef8c9879105fe4fc99417447b3a5f35d1131412ce839470bd2089fe2043f"}, - {file = "multidict-6.8.0-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:883284137e25318ed9735b742ae46341a864888fae28e8b6314c4f84da080f08"}, - {file = "multidict-6.8.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:ca52b9ec80851366197577154c862c4c4c7036ca76ae94cef5cb59c5cfeab944"}, - {file = "multidict-6.8.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:91fa75d0a693832106d98f66c849f034f21c828d14437f1fb97d3784aab89e84"}, - {file = "multidict-6.8.0-cp315-cp315t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:307c1acd812fe897e7fbe10c6758822e8c04be4e7c60a9f54901cdf8b5ab8bc3"}, - {file = "multidict-6.8.0-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5caf684986a2490628f059a99dd107b566a2d34cf947f8eb8387e0500a1f90c5"}, - {file = "multidict-6.8.0-cp315-cp315t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:77745725125d01fd613b6db043362aa7c6bfbfdb23d45dbfc3d92bf58160af62"}, - {file = "multidict-6.8.0-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8daafaa0b2eb43f76898ced78b1e0fb91b38c4fa50da516c18067f2a2d578c20"}, - {file = "multidict-6.8.0-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c68e0c0649d17c2d0339e3674e86a4aeba4a7e6b21c1e394cf947a95433b31d0"}, - {file = "multidict-6.8.0-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d8a5ac357ac283490a8d1899b0383355fd1f8634b14ba0d59e4c0dd97db85556"}, - {file = "multidict-6.8.0-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:46029e6e27a3ec0dc55b53f58df82d10f04c5e111f78248279b530bedad2c30a"}, - {file = "multidict-6.8.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:8d1046b5427dcafe6e8a0e07527dd74f1ee694006160162f53f3a17f15aad3b4"}, - {file = "multidict-6.8.0-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:f1f4a220db6ed7c8fd16b6d644ffd1f082651693204daf3275e049fadc849e39"}, - {file = "multidict-6.8.0-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:029897732a9c798737457e382bf84e8c64237eff224a90aea2639f4413c45e4e"}, - {file = "multidict-6.8.0-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:29be9fd289e9ab8f480996ea2f686e1654b80242033843cb11691688329423f1"}, - {file = "multidict-6.8.0-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:29631224698de1e42abc8fa7658d830e0aed0029785144b5832b695da5adef2f"}, - {file = "multidict-6.8.0-cp315-cp315t-musllinux_1_2_s390x.whl", hash = "sha256:962f18c59a000f30b084ea2e6b8001521bb315efd4e5f10acf9fb36f366b7882"}, - {file = "multidict-6.8.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:c60e50bc5b07faac92fd3a20fa21cc8cf3e3f7204d2867b206c73293ebc19101"}, - {file = "multidict-6.8.0-cp315-cp315t-win32.whl", hash = "sha256:fc5460940f50dff00731b4132366840ba9685286ea88ea104b661899084f3fea"}, - {file = "multidict-6.8.0-cp315-cp315t-win_amd64.whl", hash = "sha256:b367c342327717d644db4c0ddb37ceb655c84822215ea0773a3a36911b74b71d"}, - {file = "multidict-6.8.0-cp315-cp315t-win_arm64.whl", hash = "sha256:0c1c4debad7337627b86837abdf0237ca3cb3d7e17de7eab0177c263878546d4"}, - {file = "multidict-6.8.0-py3-none-any.whl", hash = "sha256:75daa15ca16d6285eb2e104b2f05ee6f8d9836c68da3ce5c85f615a0450eed0e"}, - {file = "multidict-6.8.0.tar.gz", hash = "sha256:5cd4637ce76312ba1e05eb9c5193fec231f64fee0944e135fa1e951242355b37"}, -] - [[package]] name = "packaging" version = "26.3" @@ -1332,137 +810,6 @@ files = [ dev = ["pre-commit", "tox"] testing = ["coverage", "pytest", "pytest-benchmark"] -[[package]] -name = "propcache" -version = "0.5.2" -description = "Accelerated property cache" -optional = false -python-versions = ">=3.10" -groups = ["dev"] -files = [ - {file = "propcache-0.5.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d5a81be28596d6559f6131ef33e10200de6e17643b3c74ce03f9eb103be6ae8b"}, - {file = "propcache-0.5.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:29cbaac5ea0212663e6845e04b5e188d5a6ae6dd919810ac835bf1d3b42c3f4c"}, - {file = "propcache-0.5.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6bf3be92233808fcd338eba0fb4d0b59ec5772af4f4ecfcec450d1bfc0f8b5eb"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2f8ea531c794b9d6274acd4e8d2c2ebcac590a4361d27482edd3010b79f1325e"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:decfca4c79dd53ebab484b00cc4b6717d8c369f86e74aa4ca395a64ac651495e"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4621064bbf28fa77ff64dd5d94367c04684c67d3a5bf1dff25f0cd0d98a38f3b"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b96db7141a592cbc968daf1feea83a118e6ab378af4abbc72b248c895414c22d"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1ca071adabaab6e9219924bbe00af821f1ee7de113a9eca1cdc292de3d120f4d"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e4294d04a94dcab1b3bccd8b66d962dcad411a1d19414b2a41d1445f1de32ad0"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:a0e399a2eccb91ed18721f86aa85757727400b6865c89e88934781deb9c8498b"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:823581fd5cb08b12a48bfa11fe962a7916766b6170c17b028fbdf762b85eb9bf"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:949c91d1a990cf3b2e8188dfcfb25005e0b834a06c63fa4ef9f360878ce21ecf"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:cc1177027eda740fdb152706bd215a3f124e3eea15afc39f2cb9fe351b50619e"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b05d643f944a8c3c4bd86d65ffd87bf3264b617f87791940302bc474d2ff5274"}, - {file = "propcache-0.5.2-cp310-cp310-win32.whl", hash = "sha256:8114f28879e0904748e831c3a7774261bd9e75f49be089f389a76f959dcd13fe"}, - {file = "propcache-0.5.2-cp310-cp310-win_amd64.whl", hash = "sha256:5fcb98e7598b1ee0addab320d90f65b530297a867dbfe9de52ea838077e16e3d"}, - {file = "propcache-0.5.2-cp310-cp310-win_arm64.whl", hash = "sha256:04dc2390d9edbbaef7461f33322555976ffddf0b650a038649d026358714e6c5"}, - {file = "propcache-0.5.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:74b70780220e2dd89175ca24b81b68b67c83db499ae611e7f2313cb329801c78"}, - {file = "propcache-0.5.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a4840ab0ae0216d952f4b53dc6d0b992bfc2bedbfe360bdd9b548bc184c08959"}, - {file = "propcache-0.5.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c6844ba6364fb12f403928a82cfd295ab103a2b315c77c747b2dbe4a41894ea7"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2293949b855ce597f2826452d17c2d545fb5622379c4ea6fdf525e9b8e8a2511"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0fd59b5af35f74da48d905dcbad55449ba13be91823cb05a9bd590bbf5b61660"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29f9309a2e42b0d273be006fdb4be2d6c39a47f6f57d8fb1cf9f81481df81b66"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5aaa2b923c1944ac8febd6609cb373540a5563e7cbcb0fd770f75dace2eb817b"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:66ea454f095ddf5b6b14f56c064c0941c4788be11e18d2464cf643bf7203ff67"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:95f1e3f4760d404b13c9976c0229b2b49a3c8e2c62a9ce92efdd2b11ada75e3f"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:85341b12b9d55bad0bded24cac341bb34289469e03a11f3f583ea1cc1db0326c"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:26a4dca084132874e639895c3135dfad5eb20bae209f62d1aeb31b03e601c3c0"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3b199b9b2b3d6a7edf3183ba8a9a137a22b97f7df525feb5ae1eccf026d2a9c6"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:e59bc9e66329185b93dab73f210f1a37f81cb40f321501db8017c9aea15dba27"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:552ffadf6ad409844bc5919c42a0a83d88314cedddaea0e41e80a8b8fffe881f"}, - {file = "propcache-0.5.2-cp311-cp311-win32.whl", hash = "sha256:cd416c1de191973c52ff1a12a57446bfc7642797b282d7caf2162d7d1b8aa9a0"}, - {file = "propcache-0.5.2-cp311-cp311-win_amd64.whl", hash = "sha256:44e488ef40dbb452700b2b1f8188934121f6648f52c295055662d2191959ff82"}, - {file = "propcache-0.5.2-cp311-cp311-win_arm64.whl", hash = "sha256:54adaa85a22078d1e306304a40984dc5be99d599bf3dc0a24dc98f7daeab89ab"}, - {file = "propcache-0.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:806719138ecd720339a12410fb9614ac9b2b2d3a5fdf8235d56981c36f4039ba"}, - {file = "propcache-0.5.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:db2b80ea58eab4f86b2beec3cc8b39e8ff9276ac20e96b7cce43c8ae84cd6b5a"}, - {file = "propcache-0.5.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e5cbfac9f61484f7e9f3597775500cd3ebe8274e9b050c38f9525c77c97520bf"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbc581d2814337da56222fab8dc5f161cd798a434e49bac27930aaef798e144"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:857187f381f88c8e2fa2fe56ab94879d011b883d5a2ee5a1b60a8cd2a06846d9"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:178b4a2cdaac1818e2bf1c5a99b94383fa73ea5382e032a48dec07dc5668dc42"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f328175a2cde1f0ff2c4ed8ce968b9dcfb55f3a7153f39e2957ed994da13476"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5671d09a36b06d0fd4a3da0fccbcae360e9b1570924171a15e9e0997f0249fba"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:80168e2ebe4d3ec6599d10ad8f520304ae1cad9b6c5a95372aef1b66b7bfb53a"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:45f11346f884bc47444f6e6647131055844134c3175b629f84952e2b5cd62b64"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8e778ebd44ef4f66ed60a0416b06b489687db264a9c0b3620362f26489492913"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:c0cb9ed24c8964e172768d455a38254c2dd8a552905729ce006cad3d3dda59b1"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:1d1ad32d9d4355e2be65574fd0bfd3677e7066b009cd5b9b2dee8aa6a6393b33"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c80f4ba3e8f00189165999a742ee526ebeccedf6c3f7beb0c7df821e9772435a"}, - {file = "propcache-0.5.2-cp312-cp312-win32.whl", hash = "sha256:8c7972d8f193740d9175f0998ab38717e6cd322d5935c5b0fef8c0d323fd9031"}, - {file = "propcache-0.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:d9ee8826a7d47863a08ac44e1a5f611a462eefc3a194b492da242128bec75b42"}, - {file = "propcache-0.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:2800a4a8ead6b28cccd1ec54b59346f0def7922ee1c7598e8499c733cfbb7c84"}, - {file = "propcache-0.5.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:099aaf4b4d1a02265b92a977edf00b5c4f63b3b17ac6de39b0d637c9cac0188a"}, - {file = "propcache-0.5.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:68ce1c44c7a813a7f71ea04315a8c7b330b63db99d059a797a4651bb6f69f117"}, - {file = "propcache-0.5.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fc299c129490f55f254cd90be0deca4764e36e9a7c08b4aa588479a3bbed3098"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6ae2198be502c10f09b2516e7b5d019816924bc3183a43ce792a7bd6625e6f4"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6041d31504dc1779d700e1edcfb08eea334b357620b06681a4eabb57a74e574e"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f7eabc04151c78a9f4d5bbb5f1faf571e4defeb4b585e0fe95b60ff2dbe4d3d7"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4db0ba63d693afd40d249bd93f842b5f144f8fcbb83de05660373bcf30517b1d"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1dbcf7675229b35d31abb6547d8ebc8c27a830ac3f9a794edff6254873ec7c0a"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d310c013aad2c72f1c3f2f8dd3279d460a858c551f97aeb8c63e4693cca7b4d2"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:06187263ddad280d05b4d8a8b3bb7d164cbebd469236544a42e6d9b28ac6a4fa"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3115559b8effafd63b142ea5ed53d63a16ea6469cbc63dce4ee194b42db5d853"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c60462af8e6dc30c35407c7237ea908d777b22862bbee27bc4699c0d8bcdc45a"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:40314bca9ac559716fe374094fc81c11dcc34b64fd6c585360f5775690505704"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cfa21e036ce1e1db2be04ba3b85d2df1bb1702fa01932d984c5464c665228ff4"}, - {file = "propcache-0.5.2-cp313-cp313-win32.whl", hash = "sha256:f156a3529f38063b6dbaf356e15602a7f95f8055b1295a438433a6386f10463d"}, - {file = "propcache-0.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:dfed59d0a5aeb01e242e66ff0300bc4a265a7c05f612d30016f0b60b1017d757"}, - {file = "propcache-0.5.2-cp313-cp313-win_arm64.whl", hash = "sha256:ba338430e87ceb9c8f0cf754de38a9860560261e56c00376debd628698a7364f"}, - {file = "propcache-0.5.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:a592f5f3da71c8691c788c13cb6734b6d17663d2e1cb8caddf0673d01ef8847d"}, - {file = "propcache-0.5.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6a997d0489e9668a384fcfd5061b857aa5361de73191cac204d04b889cfbbafa"}, - {file = "propcache-0.5.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:10734b5484ea113152ee25a91dccedf81631791805d2c9ccb054958e51842c94"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cafca7e56c12bb02ae16d283742bef25a61122e9dab2b5b3f2ccbe589ce32164"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f064f8d2b59177878b7615df1735cd8fe3462ed6be8c7b217d17a276489c2b7f"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f78abfa8dfc32376fd1aacf597b2f2fbbe0ea751419aee718af5d4f82537ef8c"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f7467da8a9822bf1a55336f877340c5bcbd3c482afc43a99771169f74a26dedc"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a6ddc6ac9e25de626c1f129c1b467d7ecd33ce2237d3fd0c4e429feef0a7ee1f"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:2f22cbbac9e26a8e864c0985ff1268d5d939d53d9d9411a9824279097e03a2cb"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:fc76378c62a0f04d0cd82fbb1a2cd2d7e28fcb40d5873f28a6c44e388aaa2751"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:acd2c8edba48e31e58a363b8cf4e5c7db3b04b3f9e371f601df30d9b0d244836"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:452b5065457eb9991ec5eb38ff41d6cd4c991c9ac7c531c4d5849ae473a9a13f"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:3430bb2bfe1331885c427745a751e774ee679fd4344f80b97bf879815fe8fa55"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cef6cea3922890dd6c9654971001fa797b526c16ab5e1e46c05fd6f877be7568"}, - {file = "propcache-0.5.2-cp313-cp313t-win32.whl", hash = "sha256:72d61e16dd78228b58c5d47be830ff3da7e5f139abdf0aef9d86cde1c5cf2191"}, - {file = "propcache-0.5.2-cp313-cp313t-win_amd64.whl", hash = "sha256:0958834041a0166d343b8d2cedcd8bcbaeb4fdbe0cf08320c5379f143c3be6e7"}, - {file = "propcache-0.5.2-cp313-cp313t-win_arm64.whl", hash = "sha256:6de8bd93ddde9b992cf2b2e0d796d501a19026b5b9fd87356d7d0779531a8d96"}, - {file = "propcache-0.5.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:46088abff4cba581dea21ae0467a480526cb25aa5f3c269e909f800328bc3999"}, - {file = "propcache-0.5.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fc88b26f08d634f7bc819a7852e5214f5802641ab8d9fd5326892292eee1993e"}, - {file = "propcache-0.5.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:97797ebb098e670a2f92dd66f32897e30d7615b14e7f59711de23e30a9072539"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba57fffe4ac99c5d30076161b5866336d97600769bad35cc68f7774b15298a4e"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:583c19759d9eec1e5b69e2fbef36a7d9c326041be9746cb822d335c8cedc2979"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d0326e2e5e1f3163fa306c834e48e8d490e5fae607a097a40c0648109b47ba80"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e00820e192c8dbebcafb383ebbf99030895f09905e7a0eb2e0340a0bcc2bc825"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c66afea89b1e43725731d2004732a046fe6fe955d51f952c3e95a7314a284a39"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d4dc37dec6c6cdad0b57881a5658fd14fbf53e333b1a86cf86559f190e1d9ec4"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5570dbcc97571c15f68068e529c92715a12f8d54030e272d264b377e22bd17a5"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f814362777a9f841adddb200ecdf8f5cb1e5a3c4b7a86378edbd6ccb26edd702"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:196913dea116aeb5a2ba95af4ddcb7ea85559ae07d8eee8751688310d09168c3"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:6e7b8719005dd1175be4ab1cd25e9b98659a5e0347331506ec6760d2773a7fb5"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:51f96d685ab16e88cab128cd37a52c5da540809c8b879fa047731bfcb4ad35a4"}, - {file = "propcache-0.5.2-cp314-cp314-win32.whl", hash = "sha256:cc6fc3cc62e8501d3ed62894425040d2728ecddb1ed072737a5c70bd537aa9f0"}, - {file = "propcache-0.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:81e3a30b0bb60caa22033dd0f8a3618d1d67356212514f62c57db75cb0ef410c"}, - {file = "propcache-0.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:0d2c9bf8528f135dbb805ce027567e09164f7efa51a2be07458a2c0420f292d0"}, - {file = "propcache-0.5.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:4bc8ff1feffc6a61c7002ffe84634c41b822e104990ae009f44a0834430070bb"}, - {file = "propcache-0.5.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:79aa3ff0a9b566633b642fa9caf7e21ed1c13d6feca718187873f199e1514078"}, - {file = "propcache-0.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1b31822f4474c4036bae62de9402710051d431a606d6a0f907fec79935a071aa"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13fef48778b5a2a756523fdb781326b028ca75e32858b04f2cdd19f394564917"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8b73ab70f1a3351fbc71f663b3e645af6dd0329100c353081cf69c37433fc6fe"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5538d2c13d93e4698af7e092b57bc7298fd35d1d58e656ae18f23ee0d0378e03"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cd645f03898405cabe694fb8bc35241e3a9c332ec85627584fe3de201452b335"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a473b3440261e0c60706e732b2ed2f517857344fc21bf48fdfe211e2d98eb285"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7afa37062e6650640e932e4cc9297d81f9f42d9944029cc386b8247dea4da837"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:8a90efd5777e996e42d568db9ac740b944d691e565cbfd31b2f7832f9184b2b8"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:f19bb891234d72535764d703bfed1153cc34f4214d5bd7150aee1eec9e8f4366"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:32775082acd2d807ee3db715c7770d38767b817870acfa08c29e057f3c4d5b56"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9282fb1a3bccd038da9f768b927b24a0c753e466c086b7c4f3c6982851eefb2d"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cc49723e2f60d6b32a0f0b08a3fd6d13203c07f1cd9566cfce0f12a917c967a2"}, - {file = "propcache-0.5.2-cp314-cp314t-win32.whl", hash = "sha256:2d7aa89ebca5acc98cba9d1472d976e394782f587bad6661003602a619fd1821"}, - {file = "propcache-0.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:d447bb0b3054be5818458fbb171208b1d9ff11eba14e18ca18b90cbb45767370"}, - {file = "propcache-0.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:fe67a3d11cd9b4efabfa45c3d00ffba2b26811442a73a581a94b67c2b5faccf6"}, - {file = "propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe"}, - {file = "propcache-0.5.2.tar.gz", hash = "sha256:01c4fc7480cd0598bb4b57022df55b9ca296da7fc5a8760bd8451a7e63a7d427"}, -] - [[package]] name = "pydantic" version = "2.13.5" @@ -2034,12 +1381,11 @@ version = "4.16.0" description = "Backported and Experimental Type Hints for Python 3.9+" optional = false python-versions = ">=3.9" -groups = ["main", "dev"] +groups = ["main"] files = [ {file = "typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8"}, {file = "typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5"}, ] -markers = {dev = "python_version < \"3.13\""} [[package]] name = "typing-inspection" @@ -2151,125 +1497,6 @@ pyyaml = "*" [package.extras] dev = ["doc8", "flake8", "flake8-import-order", "rstcheck[sphinx]", "ruff", "sphinx"] -[[package]] -name = "yarl" -version = "1.24.5" -description = "Yet another URL library" -optional = false -python-versions = ">=3.10" -groups = ["dev"] -files = [ - {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88f50c94e21a0a7f14042c015b0eba1881af78562e7bf007e0033e624da59750"}, - {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6efbccc3d7f75d5b03105172a8dc86d82ba4da86817952529dd93185f4a88be2"}, - {file = "yarl-1.24.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0ebfaffe1a16cb72141c8e09f18cc76856dbe58639f393a4f2b26e474b96b871"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ac73abdc7ab75610f95a8fd994c6457e87752b02a63987e188f937a1fc180f0"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4d97a951a81039050e45f04e96689b58b8243fa5e62aa14fe67cb6075300885e"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fe7b7bb170daccbba19ad33012d2b15f1e7942296fd4d45fc1b79013da8cc0f2"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:89a1bbb58e0e3f7a283653d854b1e95d65e5cfd4af224dac5f02629ec1a3e621"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7fa5e51397466ea7e98de493fa2ff1b8193cfef8a7b0f9b4842f92d342df0dba"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4103b77b8a8225e413107d2349b65eb3c1c52627b5cc5c3c4c1c6a798b218950"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f9f3e9c8a9ecffa57bef8fb4fa19e5fa4d2d8307cf6bac5b1fca5e5860f4ba00"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c0ebc836c47a6477e182169c6a476fc691d12b518894bf7dd2572f0d59f1c7ed"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:96d30286dd02679e32a39aa8f0b7498fc847fcda46cfc09df5513e82ce252440"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:fd8c81f346b58f45818d09ea11db69a8d5fd34a224b79871f6d44f12cd7977b1"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c55256dee8f4b27bfbf636c8363383c7c8db7890c7cba5217d7bd5f5f21dab6"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9f4d8cf085a4c6a40fb97ea0f46938a8df43c85d31f9d45e2a8867ea9293790d"}, - {file = "yarl-1.24.5-cp310-cp310-win_amd64.whl", hash = "sha256:240cbec09667c1fed4c6cd0060b9ec57332427d7441289a2ed8875dc9fb2b224"}, - {file = "yarl-1.24.5-cp310-cp310-win_arm64.whl", hash = "sha256:8a6987eaad834cb32dd57d9d582225f0054a5d1af706ccfbbdba735af4927e13"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2c1fe720934a16ea8e7146175cba2126f87f54912c8c5435e7f7c7a51ef808d3"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c687ed078e145f5fd53a14854beff320e1d2ab76df03e2009c98f39a0f68f39a"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:709f1efed56c4a145793c046cd4939f9959bcd818979a787b77d8e09c57a0840"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:874019bd513008b009f58657134e5d0c5e030b3559bd0553976837adf52fe966"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a4582acf7ef76482f6f511ebaf1946dae7f2e85ec4728b81a678c01df63bd723"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2cabe6546e41dabe439999a23fcb5246e0c3b595b4315b96ef755252be90caeb"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:17f57620f5475b3c69109376cc87e42a7af5db13c9398e4292772a706ff10780"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:570fec8fbd22b032733625f03f10b7ff023bc399213db15e72a7acaef28c2f4e"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5fede79c6f73ff2c3ef822864cb1ada23196e62756df53bc6231d351a49516a2"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8ccf9aca873b767977c73df497a85dbedee4ee086ae9ae49dc461333b9b79f58"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ad5d8201d310b031e6cd839d9bac2d4e5a01533ce5d3d5b50b7de1ef3af1de61"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:841f0852f48fefea3b12c9dfec00704dfa3aef5215d0e3ce564bb3d7cd8d57c6"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:9baafc71b04f8f4bb0703b21d6fc9f0c30b346c636a532ff16ec8491a5ea4b1f"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d897129df1a22b12aeed2c2c98df0785a2e8e6e0bde87b389491d0025c187077"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:dd625535328fd9882374356269227670189adfcc6a2d90284f323c05862eecbd"}, - {file = "yarl-1.24.5-cp311-cp311-win_amd64.whl", hash = "sha256:f4239bbec5a3577ddb49e4b50aeb32d8e5792098262ae2f63723f916a29b1a25"}, - {file = "yarl-1.24.5-cp311-cp311-win_arm64.whl", hash = "sha256:3ac6aff147deb9c09461b2d4bbdf6256831198f5d8a23f5d37138213090b6d8a"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d693396e5aea78db03decd60aec9ece16c9b40ba00a587f089615ff4e718a81d"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3363fcc96e665878946ad7a106b9a13eac0541766a690ef287c0232ac768b6ec"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9d399bdcfb4a0f659b9b3788bbc89babe63d9a6a65aacdf4d4e7065ff2e6316c"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90333fd89b43c0d08ac85f3f1447593fc2c66de18c3d6378d7125ea118dc7a54"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:665b0a2c463cc9423dd647e0bfd9f4ccc9b50f768c55304d5e9f80b177c1de12"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e006d3a974c4ee19512e5f058abedb6eef36a5e553c14812bdeba1758d812e6d"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e7d42c531243450ef0d4d9c172e7ed6ef052640f195629065041b5add4e058d1"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f08c7513ecef5aad65687bfdf6bc601ae9fccd04a42904501f8f7141abad9eb9"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6c95b17fe34ed802f17e205112e6e10db92275c34fee290aa9bdc55a9c724027"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:56b149b22de33b23b0c6077ab9518c6dcb538ad462e1830e68d06591ccf6e38b"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a8fe66b8f300da93798025a785a5b90b42f3810dc2b72283ff84a41aaaebc293"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:377fe3732edbaf78ee74efdf2c9f49f6e99f20e7f9d2649fda3eb4badd77d76e"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e8ffa78582120024f476a611d7befc123cee59e47e8309d470cf667d806e613b"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:daba5e594f06114e37db186efd2dd916609071e59daca901a0a2e71f02b142ce"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:65be18ec59496c13908f02a2472751d9ef840b4f3fb5726f129306bf6a2a7bba"}, - {file = "yarl-1.24.5-cp312-cp312-win_amd64.whl", hash = "sha256:a929d878fec099030c292803b31e5d5540a7b6a31e6a3cc76cb4685fc2a2f51b"}, - {file = "yarl-1.24.5-cp312-cp312-win_arm64.whl", hash = "sha256:7ce27823052e2013b597e0c738b13e7e36b8ccb9400df8959417b052ab0fd92c"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:79af890482fc94648e8cde4c68620378f7fef60932710fa17a66abc039244da2"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:46c2f213e23a04b93a392942d782eb9e413e6ef6bf7c8c53884e599a5c174dcb"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92ab3e11448f2ff7bf53c5a26eff0edc086898ec8b21fb154b85839ce1d88075"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ebb0ec7f17803063d5aeb982f3b1bd2b2f4e4fae6751226cbd6ba1fcfe9e63ff"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:82632daed195dcc8ea664e8556dc9bdbd671960fb3776bd92806ce05792c2448"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:53e549287ef628fecba270045c9701b0c564563a9b0577d24a4ec75b8ab8040f"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fcd3b77e2f17bbe4ca56ec7bcb07992647d19d0b9c05d84886dcd6f9eb810afd"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d46b86567dd4e248c6c159fcbcdcce01e0a5c8a7cd2334a0fff759d0fa075b16"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7f72c74aa99359e27a2ee8d6613fefa28b5f76a983c083074dfc2aaa4ab46213"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3f45789ce415a7ec0820dc4f82925f9b5f7732070be1dec1f5f23ec381435a24"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:6e73e7fe93f17a7b191f52ec9da9dd8c06a8fe735a1ecbd13b97d1c723bff385"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4a36f9becdd4c5c52a20c3e9484128b070b1dcfc8944c006f3a528295a359a9c"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:7bcbe0fcf850eae67b6b01749815a4f7161c560a844c769ad7b48fcd99f791c4"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:24e861e9630e0daddcb9191fb187f60f034e17a4426f8101279f0c475cd74144"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9335a099ad87287c37fe5d1a982ff392fa5efe5d14b40a730b1ec1d6a41382b4"}, - {file = "yarl-1.24.5-cp313-cp313-win_amd64.whl", hash = "sha256:2dbe06fc16bc91502bca713704022182e5729861ae00277c3a23354b40929740"}, - {file = "yarl-1.24.5-cp313-cp313-win_arm64.whl", hash = "sha256:6b8536851f9f65e7f00c7a1d49ba7f2be0ffe2c11555367fc9f50d9f842410a1"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:2729fcfc4f6a596fb0c50f32090400aa9367774ac296a00387e65098c0befa76"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ff330d3c30db4eb6b01d79e29d2d0b407a7ecad39cfd9ec993ece57396a2ec0d"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e42d75862735da90e7fc5a7b23db0c976f737113a54b3c9777a9b665e9cbff75"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a3732e66413163e72508da9eff9ce9d2846fde51fae45d3605393d3e6cd303e9"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5b8ee53be440a0cffc991a27be3057e0530122548dbe7c0892df08822fce5ede"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:af3aefa655adb5869491fa907e652290386800ae99cc50095cba71e2c6aefdca"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2120b96872df4a117cde97d270bac96aea7cc52205d305cf4611df694a487027"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:66410eb6345d467151934b49bfa70fb32f5b35a6140baa40ad97d6436abea2e9"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4af7b7e1be0a69bee8210735fe6dcfc38879adfac6d62e789d53ba432d1ffa41"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa139875ff98ab97da323cfadfaff08900d1ad42f1b5087b0b812a55c5a06373"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:0055afc45e864b92729ac7600e2d102c17bef060647e74bca75fa84d66b9ff36"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f0e466ed7511fe9d459a819edbc6c2585c0b6eabde9fa8a8947552468a7a6ef0"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f141474e85b7e54998ec5180530a7cda99ab29e282fa50e0756d89981a9b43c5"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e2935f8c39e3b03e83519292d78f075189978f3f4adc15a78144c7c8e2a1cba5"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9d1216a7f6f77836617dba35687c5b78a4170afc3c3f18fc788f785ba26565c4"}, - {file = "yarl-1.24.5-cp314-cp314-win_amd64.whl", hash = "sha256:5ba4f78df2bcc19f764a4b26a8a4f5049c110090ad5825993aacb052bf8003ad"}, - {file = "yarl-1.24.5-cp314-cp314-win_arm64.whl", hash = "sha256:9e4e16c73d717c5cf27626c524d0a2e261ad20e46932b2670f64ad5dde23e26f"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:e1ae548a9d901adca07899a4147a7c826bbcc06239d3ce9a59f57886a28a4c88"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ff405d91509d88e8d44129cd87b18d70acd1f0c1aeabd7bc3c46792b1fe2acba"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:47e98aab9d8d82ff682e7b0b5dded33bf138a32b817fcf7fa3b27b2d7c412928"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0a658a6d3fafee5c6f63c58f3e785c8c43c93fbc02bf9f2b6663f8185e0971f"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4377407001ca3c057773f44d8ddd6358fa5f691407c1ba92210bd3cf8d9e4c95"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7c0494a31a1ac5461a226e7947a9c9b78c44e1dc7185164fa7e9651557a5d9bc"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a7cff474ab7cd149765bb784cf6d78b32e18e20473fb7bda860bce98ab58e9da"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cbb833ccacdb5519eff9b8b71ee618cc2801c878e77e288775d77c3a2ced858a"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:82f75e05912e84b7a0fe57075d9c59de3cb352b928330f2eb69b2e1f54c3e1f0"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:16a2f5010280020e90f5330257e6944bc33e73593b136cc5a241e6c1dc292498"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:ffcd54362564dc1a30fb74d8b8a6e5a6b11ebd5e27266adc3b7427a21a6c9104"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0465ec8cedc2349b97a6b595ace64084a50c6e839eca40aa0626f38b8350e331"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4db9aecb141cb7a5447171b57aa1ed3a8fee06af40b992ffc31206c0b0121550"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f540c013589084679a6c7fac07096b10159737918174f5dfc5e11bf5bca4dfe6"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a61834fb15d81322d872eaafd333838ae7c9cea84067f232656f75965933d047"}, - {file = "yarl-1.24.5-cp314-cp314t-win_amd64.whl", hash = "sha256:5c88e5815a49d289e599f3513aa7fde0bc2092ff188f99c940f007f90f53d104"}, - {file = "yarl-1.24.5-cp314-cp314t-win_arm64.whl", hash = "sha256:cf139c02f5f23ef6532040a30ff662c00a318c952334f211046b8e60b7f17688"}, - {file = "yarl-1.24.5-py3-none-any.whl", hash = "sha256:a33700d13d9b7d84fd10947b09ff69fb9a792e519c8cb9764a3ca70baa6c23a7"}, - {file = "yarl-1.24.5.tar.gz", hash = "sha256:e81b83143bee16329c23db3c1b2d82b29892fcbcb849186d2f6e98a5abe9a57f"}, -] - -[package.dependencies] -idna = ">=2.0" -multidict = ">=4.0" -propcache = ">=0.2.1" - [[package]] name = "zensical" version = "0.1.1" @@ -2315,4 +1542,4 @@ files = [ [metadata] lock-version = "2.1" python-versions = "^3.11" -content-hash = "7eb028fd5910ecc88e6b348d1e5fb463fab817f017f6de3538ea837f6e532389" +content-hash = "aa9f6fed6b2d53f7578656e1e8154c9c7b66a9cb852dabe895864aa8e8ca083b" diff --git a/pyproject.toml b/pyproject.toml index 9da3f8a4b..8911329e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,7 +60,6 @@ requests = "^2.32" pyoxigraph = "0.5.13" [tool.poetry.group.dev.dependencies] -linkcheckmd = "^1.4.0" rumdl = "^0.0.194" pytest = "^9.1.1" yamllint = "^1.38.0" diff --git a/tests/test_update_di_reference.py b/tests/test_update_di_reference.py index 0ee2cdcfa..cde14d25b 100644 --- a/tests/test_update_di_reference.py +++ b/tests/test_update_di_reference.py @@ -9,6 +9,7 @@ get_plugin_descriptions, resolve_related_plugin_links, create_plugin_markdown, + create_umbrella_pages, build_plugin_paths, validate_related_plugin_references, ) @@ -265,3 +266,28 @@ def test_create_plugin_markdown_renders_empty_list_for_untagged_plugin(tmp_path) written = (tmp_path / "aggregator" / "average.md").read_text() front_matter = written.split("---")[1] assert yaml.safe_load(front_matter)["tags"] == [] + + +def test_create_umbrella_pages_writes_pages_files_as_yaml_documents(tmp_path): + """Test that every generated .pages file starts with the document marker and ends with a newline, as yamllint requires""" + plugins = { + "transformer": [_make_plugin("regexExtract")], + "dataset": [_make_plugin("json", plugin_type="dataset", main_category="File")], + } + (tmp_path / "transformer" / "Extract").mkdir(parents=True) + (tmp_path / "dataset").mkdir() + + create_umbrella_pages(plugins, tmp_path) + + pages_files = sorted(tmp_path.rglob(".pages")) + assert [p.relative_to(tmp_path).as_posix() for p in pages_files] == [ + ".pages", + "dataset/.pages", + "transformer/.pages", + "transformer/Extract/.pages", + ] + for pages_file in pages_files: + content = pages_file.read_text() + assert content.startswith("---\n") + assert content.endswith("\n") and not content.endswith("\n\n") + assert "nav" in yaml.safe_load(content) diff --git a/tools/update_di_reference.py b/tools/update_di_reference.py index b4f75187b..55f1ebf8d 100644 --- a/tools/update_di_reference.py +++ b/tools/update_di_reference.py @@ -37,6 +37,12 @@ def code_span(match: re.Match) -> str: return f"`` {content} ``" if "`" in content else f"`{content}`" +def write_pages_file(path: Path, content: str) -> None: + """Write a .pages file as a YAML document that passes yamllint: document start marker, final newline.""" + click.echo(f"Create .pages file {path}") + path.write_text(f"---\n{content}\n", encoding="utf-8") + + def stripped_single_line(value: str) -> str: return re.sub(r"\s+", " ", FENCED_BLOCK.sub(code_span, value)).strip() @@ -243,17 +249,16 @@ def create_umbrella_pages(plugins: dict[str, list[PluginDescription]], base_dir: f.write(reference_base_template.render()) # Create the main .pages file - reference_base_pages_file = base_dir / f".pages" - with reference_base_pages_file.open("w", encoding="utf-8") as f: - click.echo(f"Create the main .pages file: {reference_base_pages_file}") - content = """nav: + write_pages_file( + base_dir / ".pages", + """nav: - "Task and Operator Reference": index.md - "Aggregators": aggregator - "Custom Workflow Tasks": customtask - "Datasets": dataset - "Distance Measures": distancemeasure - - "Transformers": transformer""" - f.write(content) + - "Transformers": transformer""", + ) for type_id, plugins_of_type in plugins.items(): if type_id == "transformer": @@ -284,17 +289,13 @@ def create_umbrella_pages(plugins: dict[str, list[PluginDescription]], base_dir: category_plugins = [plugin for plugin in plugins[type_id] if plugin.main_category == category] for plugin in category_plugins: sub_pages_content += f"\n - \"{plugin.title}\": {plugin.pluginId}.md" - with sub_pages_file.open("w", encoding="utf-8") as f: - click.echo(f"Create .pages file {sub_pages_file}") - f.write(sub_pages_content) + write_pages_file(sub_pages_file, sub_pages_content) else: for plugin in plugins_of_type: if plugin.is_deprecated: continue pages_content += f"\n - \"{plugin.title}\": {plugin.pluginId}.md" - with pages_file.open("w", encoding="utf-8") as f: - click.echo(f"Create .pages file {pages_file}") - f.write(pages_content) + write_pages_file(pages_file, pages_content) def build_plugin_paths(plugins: dict[str, list[PluginDescription]]) -> dict[str, str]: """Map every non-deprecated plugin's ID to the path its own page will have.