From 73a5899d52044a126857a5b8c367cd575eee24ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Fri, 31 Jul 2026 10:44:23 +0200 Subject: [PATCH 01/45] feat(engine): introduce new raw object field subtype that disables indexing & parsing --- .../src/database/engine-mapping-generator.ts | 4 ++++ .../opencti-graphql/src/database/engine.ts | 3 ++- .../src/schema/attribute-definition.ts | 3 ++- .../src/schema/schema-attributes.ts | 17 ++++++++++++++--- 4 files changed, 22 insertions(+), 5 deletions(-) diff --git a/opencti-platform/opencti-graphql/src/database/engine-mapping-generator.ts b/opencti-platform/opencti-graphql/src/database/engine-mapping-generator.ts index 32b318e70529..3c2c745aba04 100644 --- a/opencti-platform/opencti-graphql/src/database/engine-mapping-generator.ts +++ b/opencti-platform/opencti-graphql/src/database/engine-mapping-generator.ts @@ -48,6 +48,10 @@ const attributeMappingGenerator = (entityAttribute: AttributeDefinition, engine: if (entityAttribute.format === 'flat') { return { type: engine instanceof ElkClient ? 'flattened' : 'flat_object' }; } + // For non-indexed objects + if (entityAttribute.format === 'raw') { + return { type: 'object', enabled: false }; + } // For standard object const properties: Record = {}; for (let i = 0; i < entityAttribute.mappings.length; i += 1) { diff --git a/opencti-platform/opencti-graphql/src/database/engine.ts b/opencti-platform/opencti-graphql/src/database/engine.ts index 2a6c968f18a3..12b1e1c052c4 100644 --- a/opencti-platform/opencti-graphql/src/database/engine.ts +++ b/opencti-platform/opencti-graphql/src/database/engine.ts @@ -131,6 +131,7 @@ import { isDateNumericOrBooleanAttribute, isNumericAttribute, isObjectFlatAttribute, + isObjectRawAttribute, schemaAttributesDefinition, validateDataBeforeIndexing, } from '../schema/schema-attributes'; @@ -4493,7 +4494,7 @@ export const prepareElementForIndexing = async (element: Record) => thing[key] = typeof value === 'boolean' ? value : value?.toLowerCase() === 'true'; } else if (isNumericAttribute(key)) { thing[key] = isNotEmptyField(value) ? Number(value) : undefined; - } else if (R.is(Object, value) && Object.keys(value).length > 0) { // For complex object, prepare inner elements + } else if (R.is(Object, value) && Object.keys(value).length > 0 && !isObjectRawAttribute(key)) { // For complex object, prepare inner elements thing[key] = await prepareElementForIndexing(value); } else if (R.is(String, value)) { // For string, trim by default thing[key] = value.trim(); diff --git a/opencti-platform/opencti-graphql/src/schema/attribute-definition.ts b/opencti-platform/opencti-graphql/src/schema/attribute-definition.ts index c82b40665b9c..e38a8e2b14b7 100644 --- a/opencti-platform/opencti-graphql/src/schema/attribute-definition.ts +++ b/opencti-platform/opencti-graphql/src/schema/attribute-definition.ts @@ -82,11 +82,12 @@ export type VocabAttribute = { type: 'string'; format: 'vocabulary'; vocabularyC export type JsonAttribute = { type: 'string'; format: 'json'; attrRawIds?: GetRawIdsFn; representative?: RepresentativeFn; multiple: false; schemaDef?: Record } & BasicDefinition; export type ObjectDefinition = { type: 'object'; attrRawIds?: GetRawIdsFn; representative?: RepresentativeFn } & BasicDefinition; export type FlatObjectAttribute = { type: 'object'; format: 'flat' } & ObjectDefinition; +export type RawObjectAttribute = { type: 'object'; format: 'raw' } & ObjectDefinition; export type ObjectAttribute = { type: 'object'; format: 'standard' } & BasicObjectDefinition; export type NestedObjectAttribute = { type: 'object'; format: 'nested' } & BasicObjectDefinition; export type RefAttribute = { type: 'ref'; attrRawIds?: GetRawIdsFn; databaseName: string; stixName: string; isRefExistingForTypes: Checker; datable?: boolean; toTypes: string[] } & BasicDefinition; export type StringAttribute = IdAttribute | TextAttribute | EnumAttribute | VocabAttribute | JsonAttribute; -export type ComplexAttribute = FlatObjectAttribute | ObjectAttribute | NestedObjectAttribute; +export type ComplexAttribute = FlatObjectAttribute | ObjectAttribute | NestedObjectAttribute | RawObjectAttribute; export type ComplexAttributeWithMappings = ObjectAttribute | NestedObjectAttribute; export type AttributeDefinition = NumericAttribute | DateAttribute | BooleanAttribute diff --git a/opencti-platform/opencti-graphql/src/schema/schema-attributes.ts b/opencti-platform/opencti-graphql/src/schema/schema-attributes.ts index 58cd5ec268e4..2d907516deee 100644 --- a/opencti-platform/opencti-graphql/src/schema/schema-attributes.ts +++ b/opencti-platform/opencti-graphql/src/schema/schema-attributes.ts @@ -25,6 +25,9 @@ const isDateAttributeDefinition = (schemaDef: AttributeDefinition) => schemaDef. const isNonFlatObjectAttributeDefinition = (schemaDef: AttributeDefinition): schemaDef is ComplexAttributeWithMappings => { // handy typeguard return schemaDef.type === 'object' && schemaDef.format !== 'flat'; }; +const isRawObjectAttributeDefinition = (schemaDef: AttributeDefinition): schemaDef is ComplexAttributeWithMappings => { // handy typeguard + return schemaDef.type === 'object' && schemaDef.format === 'raw'; +}; /** * Returns the attribute definition for a given dotted path inside the given AttributeDefinition, @@ -145,7 +148,7 @@ export const schemaAttributesDefinition = { }); } // Check sortBy on object - if (attribute.type === 'object' && attribute.format !== 'flat' && attribute.sortBy) { + if (attribute.type === 'object' && attribute.format !== 'flat' && attribute.format !== 'raw' && attribute.sortBy) { const correspondingMapping = getAttributeMappingFromPath(attribute.sortBy.path, attribute); if (correspondingMapping.type !== attribute.sortBy.type) { throw UnsupportedError('You can\'t define a sortBy with path and type that do not match the corresponding mapping', { @@ -155,7 +158,7 @@ export const schemaAttributesDefinition = { } } let registeredAttribute: AttributeDefinition = { ...attribute }; - if (registeredAttribute.type === 'object' && registeredAttribute.format !== 'flat' && registeredAttribute.mappings.length > 0) { + if (registeredAttribute.type === 'object' && registeredAttribute.format !== 'flat' && registeredAttribute.format !== 'raw' && registeredAttribute.mappings.length > 0) { registeredAttribute = { ...registeredAttribute, // filter feature flagged attributes @@ -294,7 +297,11 @@ export const isObjectFlatAttribute = (k: string): boolean => { if (!definition) return false; return definition.type === 'object' && definition.format === 'flat'; }; - +export const isObjectRawAttribute = (k: string): boolean => { + const definition = schemaAttributesDefinition.getAttributeByName(k.split('.')[0]); + if (!definition) return false; + return definition.type === 'object' && definition.format === 'raw'; +}; // -- MULTIPLE -- export const isMultipleAttribute = (entityType: string, k: string): boolean => ( @@ -332,6 +339,10 @@ const validateInputAgainstSchema = (input: any, schemaDef: AttributeDefinition) throw FunctionalError(`Validation against schema failed on attribute [${schemaDef.name}]: value must be an object`); } + if (isRawObjectAttributeDefinition(schemaDef)) { + return; // raw objects are not recursively validated + } + const inputValues = Array.isArray(input) ? input : [input]; inputValues.forEach((value) => { // check the value adhere to its mapping From f97139fd8080b3b460583632160273b3e0f16d50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Thu, 13 Aug 2026 13:35:03 +0200 Subject: [PATCH 02/45] feat(connector): persist snapshot of catalog contract in new manager_contract connector field --- opencti-platform/opencti-front/vite.config.ts | 1 + .../config/schema/opencti.graphql | 2 +- .../src/database/repository.js | 46 ++-- .../src/domain/connector-migration.ts | 23 +- .../opencti-graphql/src/domain/connector.ts | 25 ++- .../opencti-graphql/src/http/httpPlatform.js | 4 + ...ed-catalog-contract-in-connector-entity.ts | 76 +++++++ .../internalObject-registrationAttributes.ts | 4 + .../src/modules/catalog/catalog-domain.ts | 68 +++++- .../src/modules/catalog/catalog-http.ts | 46 ++++ .../modules/catalog/catalog-logo-storage.ts | 169 +++++++++++++++ .../src/modules/catalog/catalog-types.ts | 91 +++++++- .../src/modules/catalog/catalog.graphql | 4 +- .../src/modules/catalog/catalog.ts | 99 +++++++++ .../opencti-graphql/src/types/connector.d.ts | 7 +- .../opencti-graphql/src/utils/data-url.ts | 32 +++ .../02-resolvers/connector-composer-test.ts | 197 +++++++++++++++++- 17 files changed, 853 insertions(+), 41 deletions(-) create mode 100644 opencti-platform/opencti-graphql/src/migrations/1786264797376-embed-catalog-contract-in-connector-entity.ts create mode 100644 opencti-platform/opencti-graphql/src/modules/catalog/catalog-http.ts create mode 100644 opencti-platform/opencti-graphql/src/modules/catalog/catalog-logo-storage.ts create mode 100644 opencti-platform/opencti-graphql/src/modules/catalog/catalog.ts create mode 100644 opencti-platform/opencti-graphql/src/utils/data-url.ts diff --git a/opencti-platform/opencti-front/vite.config.ts b/opencti-platform/opencti-front/vite.config.ts index 2902a648bfb1..e0bef487e167 100644 --- a/opencti-platform/opencti-front/vite.config.ts +++ b/opencti-platform/opencti-front/vite.config.ts @@ -77,6 +77,7 @@ export default defineConfig(({ mode, command }) => { [`${basePath}/logout`]: backProxy(), [`${basePath}/stream`]: backProxy(), [`${basePath}/storage`]: backProxy(), + [`${basePath}/catalog`]: backProxy(), [`${basePath}/schema`]: backProxy(), '^/.*/embedded/.*': backProxy(), [`${basePath}/taxii2`]: backProxy(), diff --git a/opencti-platform/opencti-graphql/config/schema/opencti.graphql b/opencti-platform/opencti-graphql/config/schema/opencti.graphql index e0c94f1a27c0..f3ae008f553d 100644 --- a/opencti-platform/opencti-graphql/config/schema/opencti.graphql +++ b/opencti-platform/opencti-graphql/config/schema/opencti.graphql @@ -2249,7 +2249,7 @@ type Connector implements BasicObject & InternalObject { manager_current_status: String manager_requested_status: String manager_contract_image: String - manager_contract_definition: String + manager_contract_definition: String # JSON.Stringified GraphqlCatalogContract item manager_contract_excerpt: ManagerContractExcerpt manager_contract_configuration: [ManagerContractConfiguration!] manager_connector_logs: [String!] diff --git a/opencti-platform/opencti-graphql/src/database/repository.js b/opencti-platform/opencti-graphql/src/database/repository.js index 603446259b26..f6bcdc0a1d19 100644 --- a/opencti-platform/opencti-graphql/src/database/repository.js +++ b/opencti-platform/opencti-graphql/src/database/repository.js @@ -9,11 +9,11 @@ import { BUILTIN_NOTIFIERS_CONNECTORS } from '../modules/notifier/notifier-stati import { builtInConnector, builtInConnectorsRuntime } from '../connector/connector-domain'; import { ENTITY_TYPE_PLAYBOOK } from '../modules/playbook/playbook-types'; import { shortHash } from '../schema/schemaUtils'; -import { encryptValue, getSupportedContractsByImage } from '../modules/catalog/catalog-domain'; +import { encryptValue, mapContractEntityFieldsToGraphqlCatalogContract } from '../modules/catalog/catalog-domain'; import { ENTITY_TYPE_PIR } from '../modules/pir/pir-types'; import { getEntitiesMapFromCache } from './cache'; import { SYSTEM_USER } from '../utils/access'; -import conf, { booleanConf, logApp } from '../config/conf'; +import conf, { booleanConf } from '../config/conf'; import { ConnectorPriorityGroup } from '../generated/graphql'; import { injectProxyConfiguration } from '../config/proxy-config'; import { getPlatformCrypto } from '../utils/platformCrypto'; @@ -22,7 +22,7 @@ import { memoize } from '../utils/memoize'; import { addUserTokenByAdmin, revokeUserTokenByAdmin } from '../modules/user/user-domain'; import { getClientBase } from './redis'; import { lockResources } from '../lock/master-lock'; -import { LockTimeoutError, TYPE_LOCK_ERROR } from '../config/errors'; +import { FunctionalError, LockTimeoutError, TYPE_LOCK_ERROR } from '../config/errors'; const getJWTKeyPair = memoize(async () => { const factory = await getPlatformCrypto(); @@ -76,28 +76,22 @@ export const connector = async (context, user, id) => { }; export const computeManagerConnectorContract = async (_context, _user, cn) => { - const contracts = await getSupportedContractsByImage(); - const contract = contracts.get(cn.manager_contract_image); - return contract ? JSON.stringify(contract) : contract; -}; - -export const computeManagerConnectorExcerpt = async (_context, _user, cn) => { - if (!cn.manager_contract_image) { + if (!cn.manager_contract) { return null; } + return JSON.stringify(mapContractEntityFieldsToGraphqlCatalogContract(cn.manager_contract)); +}; - const contracts = await getSupportedContractsByImage(); - const contract = contracts.get(cn.manager_contract_image); - - if (!contract) { - logApp.warn('No contract found for', { connectorName: cn.name }); +export const computeManagerConnectorExcerpt = async (_context, _user, cn) => { + if (!cn.manager_contract) { return null; } + const managerContract = cn.manager_contract; return { - title: contract.title, - slug: contract.slug, - logo: contract.logo, + title: managerContract.title, + slug: managerContract.slug, + logo: managerContract.logo_uri ?? '', }; }; @@ -171,10 +165,18 @@ export const computeManagerConnectorConfiguration = async (context, _, cn, { wit }; export const computeManagerConnectorImage = async (cn) => { - const contracts = await getSupportedContractsByImage(); - const contract = contracts.get(cn.manager_contract_image); - if (!contract) return ''; - return isNotEmptyField(cn.manager_contract_image) ? `${cn.manager_contract_image}:${contract.container_version}` : null; + if (!cn.manager_contract) { + return ''; + } + const managerContract = cn.manager_contract; + if (!managerContract.image || !managerContract.version) { + throw FunctionalError('Invalid manager contract snapshot', { + connectorId: cn.id ?? cn.internal_id, + image: managerContract.image, + version: managerContract.version, + }); + } + return `${managerContract.image}:${managerContract.version}`; }; export const computeManagerContractHash = async (context, user, cn) => { diff --git a/opencti-platform/opencti-graphql/src/domain/connector-migration.ts b/opencti-platform/opencti-graphql/src/domain/connector-migration.ts index 45b1bcaf5683..46b2a7051259 100644 --- a/opencti-platform/opencti-graphql/src/domain/connector-migration.ts +++ b/opencti-platform/opencti-graphql/src/domain/connector-migration.ts @@ -7,13 +7,16 @@ import { completeConnector, connector, connectors } from '../database/repository import type { Connector, ConnectorContractConfiguration, ContractConfigInput } from '../generated/graphql'; import { publishUserAction } from '../listener/UserActionListener'; import { addConnectorDeployedCount } from '../manager/telemetryManager'; -import { computeConnectorTargetContract, findContractByContainerImage } from '../modules/catalog/catalog-domain'; +import { computeConnectorTargetContract, findContractByContainerImage, mapContractDtoV0ToContractEntityFields } from '../modules/catalog/catalog-domain'; import { ABSTRACT_INTERNAL_OBJECT } from '../schema/general'; import { ENTITY_TYPE_CONNECTOR, ENTITY_TYPE_CONNECTOR_MANAGER } from '../schema/internalObject'; import type { BasicStoreEntityConnectorManager } from '../types/connector'; import type { AuthContext, AuthUser } from '../types/user'; import { isServiceAccountUser } from '../utils/access'; import { resolveUserByIdFromCache, userEditField } from './user'; +import type { CatalogContractDtoV0 } from '../modules/catalog/catalog-types'; +import { now } from '../utils/format'; +import { listCatalogContractLogos, storeCatalogContractLogo } from '../modules/catalog/catalog-logo-storage'; type ConfigInput = { key: string; @@ -22,6 +25,7 @@ type ConfigInput = { type MappedKey = { key: string; + sourceKey: string; value: string; type: string; required: boolean; @@ -89,6 +93,7 @@ const categorizeKeys = ( if (value !== null && value !== undefined) { mapped.push({ key: schemaKey, + sourceKey: keyUpper, value: String(value), type: propSchema.type, required: requiredKeys.includes(schemaKey), @@ -144,9 +149,9 @@ export const assessConnectorMigration = async (context: AuthContext, user: AuthU throw FunctionalError('Contract not found', { container_image: containerImage }); } - let contract; + let contract: CatalogContractDtoV0; try { - contract = JSON.parse(contractData.contract); + contract = JSON.parse(contractData.contract) as CatalogContractDtoV0; } catch { throw FunctionalError('Cannot parse contract found'); } @@ -185,7 +190,7 @@ export const assessConnectorMigration = async (context: AuthContext, user: AuthU connector_id: connectorId, connector_name: existingConnector.name, connector_type: existingConnector.connector_type, - contract_slug: contract.contract_slug, + contract_slug: contract.slug, contract_title: contract.title, contract_image: contract.container_image, summary: { @@ -193,6 +198,7 @@ export const assessConnectorMigration = async (context: AuthContext, user: AuthU mapped_keys: mapped.length, ignored_keys: ignored.length, missing_mandatory_keys: missingMandatory.length, + assessment_date: now(), missing_optional_keys: missing.length - missingMandatory.length, can_migrate: missingMandatory.length === 0, configuration_provided: configuration !== null, @@ -358,10 +364,19 @@ export const migrateConnectorToManaged = async ( ); } + const existingLogos = await listCatalogContractLogos(); + const logoStorageResult = await storeCatalogContractLogo(contract, existingLogos); + const managerContract = mapContractDtoV0ToContractEntityFields({ + catalogId: contractData.catalog_id, + contractDto: contract, + logoUri: logoStorageResult.logoUri, + }); + const managedConnectorData: any = { title: existingConnector.name, catalog_id: contractData.catalog_id, manager_contract_image: contract.container_image, + manager_contract: managerContract, manager_contract_configuration: filteredConfigurations, manager_requested_status: 'stopped', }; diff --git a/opencti-platform/opencti-graphql/src/domain/connector.ts b/opencti-platform/opencti-graphql/src/domain/connector.ts index 550025a56234..61f6e4c2696d 100644 --- a/opencti-platform/opencti-graphql/src/domain/connector.ts +++ b/opencti-platform/opencti-graphql/src/domain/connector.ts @@ -55,7 +55,7 @@ import { isCompatibleVersionWithMinimal } from '../utils/version'; import { extractEntityRepresentativeName } from '../database/entity-representative'; import type { BasicStoreCommon, StoreEntity } from '../types/store'; import { addConnectorDeployedCount, addWorkbenchDraftConvertionCount, addWorkbenchValidationCount } from '../manager/telemetryManager'; -import { computeConnectorTargetContract, getSupportedContractsByImage } from '../modules/catalog/catalog-domain'; +import { computeConnectorTargetContract, getSupportedContractsByImage, mapContractDtoV0ToContractEntityFields } from '../modules/catalog/catalog-domain'; import { getEntitiesMapFromCache } from '../database/cache'; import { createOnTheFlyUser } from '../modules/user/user-domain'; @@ -67,6 +67,7 @@ import { extractContentFrom } from '../utils/fileToContent'; import type { FileHandle } from 'fs/promises'; import { encryptSynchronizerCredential } from './connector-sync-crypto'; import { verifyIngestionUri } from '../modules/ingestion/ingestion-common'; +import { listCatalogContractLogos, storeCatalogContractLogo } from '../modules/catalog/catalog-logo-storage'; const MINIMAL_SYNCHRONIZER_COMPATIBLE_VERSION = '6.9.6'; // Sanitize name for K8s/Docker @@ -89,6 +90,26 @@ const sanitizeContainerName = (label: string): string => { return sanitized; }; +const computeManagerContractSnapshot = async ( + catalogId: string, + contractDto: any, +) => { + const existingLogos = await listCatalogContractLogos(); + const logoStorageResult = await storeCatalogContractLogo(contractDto, existingLogos); + if (logoStorageResult.result === 'failed') { + logApp.warn('[CONNECTOR] Failed to store manager contract logo', { + catalogId, + slug: contractDto.slug, + cause: logoStorageResult.error?.message, + }); + } + return mapContractDtoV0ToContractEntityFields({ + catalogId, + contractDto, + logoUri: logoStorageResult.logoUri, + }); +}; + // region connectors export const connectorForWork = async (context: AuthContext, user: AuthUser, id: string) => { const work = await elLoadById(context, user, id, { type: ENTITY_TYPE_WORK, indices: READ_INDEX_HISTORY }); @@ -303,6 +324,7 @@ export const managedConnectorAdd = async ( } // Create connector + const managerContract = await computeManagerContractSnapshot(input.catalog_id, targetContract); const connectorToCreate: any = { title: input.name, name: sanitizedName, @@ -311,6 +333,7 @@ export const managedConnectorAdd = async ( connector_user_id: connectorUser.id, manager_contract_image: input.manager_contract_image, manager_contract_configuration: contractConfigurations, + manager_contract: managerContract, manager_requested_status: 'stopped', connector_state_timestamp: now(), built_in: false, diff --git a/opencti-platform/opencti-graphql/src/http/httpPlatform.js b/opencti-platform/opencti-graphql/src/http/httpPlatform.js index c589cb16c712..d2dda29cf020 100644 --- a/opencti-platform/opencti-graphql/src/http/httpPlatform.js +++ b/opencti-platform/opencti-graphql/src/http/httpPlatform.js @@ -52,6 +52,7 @@ import { CERT_PROVIDER } from '../modules/authenticationProvider/provider-cert'; import { HEADERS_PROVIDER } from '../modules/authenticationProvider/provider-headers'; import { AuthenticationProviderError } from '../modules/authenticationProvider/providers-logger'; import { buildDefaultHelmetParameters, buildPublicHelmetParameters } from './httpUtils'; +import { CATALOG_LOGO_VIEW_PATH, handleCatalogLogoViewRequest } from '../modules/catalog/catalog-http'; const publicDir = DEV_MODE ? '../opencti-front/dist' : 'public'; @@ -300,6 +301,9 @@ const createApp = async (app, schema) => { } }); + // -- Catalog logo view + app.get(`${basePath}${CATALOG_LOGO_VIEW_PATH}`, handleCatalogLogoViewRequest); + // -- embedded loader const uuidPattern = '[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}'; const embeddedFileGetPath = new RegExp(`${basePath}/(.*)/(${uuidPattern})/(.*)embedded/(.*)$`, 'i'); diff --git a/opencti-platform/opencti-graphql/src/migrations/1786264797376-embed-catalog-contract-in-connector-entity.ts b/opencti-platform/opencti-graphql/src/migrations/1786264797376-embed-catalog-contract-in-connector-entity.ts new file mode 100644 index 000000000000..3dae5e3ae9ef --- /dev/null +++ b/opencti-platform/opencti-graphql/src/migrations/1786264797376-embed-catalog-contract-in-connector-entity.ts @@ -0,0 +1,76 @@ +import { logMigration } from '../config/conf'; +import { READ_INDEX_INTERNAL_OBJECTS } from '../database/utils'; +import { FilterMode } from '../generated/graphql'; +import { fullEntitiesList } from '../database/middleware-loader'; +import { executionContext, SYSTEM_USER } from '../utils/access'; +import { ENTITY_TYPE_CONNECTOR } from '../schema/internalObject'; +import { patchAttribute } from '../database/middleware'; +import type { BasicStoreEntityConnector } from '../types/connector'; +import openCtiManifest from '../__generated__/opencti-manifest.json'; +import type { CatalogContract, CatalogDefinition } from '../modules/catalog/catalog-types'; +import { mapContractDtoV0ToContractEntityFields } from '../modules/catalog/catalog-domain'; +import { listCatalogContractLogos, storeCatalogContractLogo } from '../modules/catalog/catalog-logo-storage'; + +const message = '[MIGRATION] managed connectors contract snapshot'; + +export const up = async (next: (error?: Error) => void) => { + logMigration.info(`${message} > started`); + const context = executionContext('migration'); + + const filigranCatalog = openCtiManifest as CatalogDefinition; + + // Load all managed connectors which have their contract definition in the embedded catalog manifest + const managedConnectorsArgs = { + indices: [READ_INDEX_INTERNAL_OBJECTS], + filters: { mode: FilterMode.And, filters: [{ key: ['catalog_id'], values: [filigranCatalog.id] }], filterGroups: [] }, + }; + const managedConnectors = await fullEntitiesList(context, SYSTEM_USER, [ENTITY_TYPE_CONNECTOR], managedConnectorsArgs); + logMigration.info(`${message} > ${managedConnectors.length} managed connector(s) to evaluate`); + + // Read the contracts from the embedded catalog file + const contractsByImage: Record = {}; + filigranCatalog.contracts.forEach((contract: CatalogContract) => { + contractsByImage[contract.container_image] = contract; + }); + + const storedLogos = managedConnectors.length > 0 ? (await listCatalogContractLogos()) : new Set(); + for (let i = 0; i < managedConnectors.length; i += 1) { + const connector = managedConnectors[i]; + const patch: Pick = {}; + if (!connector.manager_contract && connector.manager_contract_image) { + const contractDto = contractsByImage[connector.manager_contract_image]; + if (!contractDto) { + logMigration.info(`${message} > connector ${connector.id} missing catalog contract ${connector.manager_contract_image}`); + } else { + const result = await storeCatalogContractLogo(contractDto, storedLogos); + if (result.result === 'failed') { + logMigration.info(`${message} > failed to store logo into file storage for connector ${connector.id} (${result.error?.message ?? 'Unknown error'})`); + } else if (result.result === 'success') { + if (result.existed) { + logMigration.info(`${message} > deduped logo file for connector ${connector.id}`); + } else { + storedLogos.add(result.filename); + } + } + patch.manager_contract = mapContractDtoV0ToContractEntityFields({ + catalogId: filigranCatalog.id, + contractDto, + logoUri: result.logoUri, + }); + } + } + if (!connector.manager_upgrade_strategy) { + patch.manager_upgrade_strategy = 'latest'; + } + if (Object.keys(patch).length > 0) { + await patchAttribute(context, SYSTEM_USER, connector.id, ENTITY_TYPE_CONNECTOR, patch); + logMigration.info(`${message} > connector ${connector.id} patched`); + } + } + logMigration.info(`${message} > done`); + next(); +}; + +export const down = async (next: (error?: Error) => void) => { + next(); +}; diff --git a/opencti-platform/opencti-graphql/src/modules/attributes/internalObject-registrationAttributes.ts b/opencti-platform/opencti-graphql/src/modules/attributes/internalObject-registrationAttributes.ts index 0cfd596a4fba..03c55002ab75 100644 --- a/opencti-platform/opencti-graphql/src/modules/attributes/internalObject-registrationAttributes.ts +++ b/opencti-platform/opencti-graphql/src/modules/attributes/internalObject-registrationAttributes.ts @@ -41,6 +41,7 @@ import { schemaAttributesDefinition } from '../../schema/schema-attributes'; import { ENTITY_TYPE_MARKING_DEFINITION } from '../../schema/stixMetaObject'; import { ENTITY_TYPE_IDENTITY_ORGANIZATION } from '../organization/organization-types'; import { ENTITY_TYPE_PIR } from '../pir/pir-types'; +import { CATALOG_CONTRACT_MAPPINGS } from '../catalog/catalog'; const HistoryDefinition: AttributeDefinition[] = [ { name: 'event_type', label: 'Event type', type: 'string', format: 'enum', values: EVENT_TYPE_VALUES, editDefault: false, mandatoryType: 'internal', multiple: false, upsert: false, isFilterable: true }, @@ -464,6 +465,9 @@ const internalObjectsAttributes: { [k: string]: Array> { name: 'manager_requested_status', label: 'Connector manager requested status', type: 'string', format: 'short', mandatoryType: 'no', editDefault: false, multiple: false, upsert: false, isFilterable: true }, { name: 'manager_contract_image', label: 'Connector manager image', type: 'string', format: 'short', mandatoryType: 'no', editDefault: false, multiple: false, upsert: false, isFilterable: true }, { name: 'manager_contract_configuration', label: 'Connector manager', type: 'object', format: 'flat', mandatoryType: 'no', editDefault: false, multiple: true, upsert: false, isFilterable: true }, + { name: 'manager_contract', label: 'Connector manager contract', type: 'object', format: 'standard', mandatoryType: 'no', editDefault: false, multiple: false, upsert: false, isFilterable: false, mappings: CATALOG_CONTRACT_MAPPINGS }, + { name: 'manager_upgrade_strategy', label: 'Connector upgrade strategy', type: 'string', format: 'enum', values: ['latest'], mandatoryType: 'no', editDefault: false, multiple: false, upsert: false, isFilterable: false }, + // endregion ], [ENTITY_TYPE_CONNECTOR_MANAGER]: [ diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-domain.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-domain.ts index ed6215dab673..3e8546f96fa5 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-domain.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-domain.ts @@ -2,7 +2,14 @@ import Ajv from 'ajv'; import addFormats from 'ajv-formats'; import crypto from 'crypto'; import type { AuthContext, AuthUser } from '../../types/user'; -import { type CatalogContract, type CatalogDefinition, type CatalogType } from './catalog-types'; +import { + type CatalogContract, + type CatalogContractDtoV0, + type CatalogContractEntityFields, + type CatalogDefinition, + type CatalogType, + type GraphqlCatalogContract, +} from './catalog-types'; import { isEmptyField } from '../../database/utils'; import { UnsupportedError } from '../../config/errors'; import { idGenFromData } from '../../schema/identifier'; @@ -493,3 +500,62 @@ export const findContractBySlug = (context: AuthContext, user: AuthUser, contrac export const findContractByContainerImage = (context: AuthContext, user: AuthUser, containerImage: string) => { return findContract(context, user, (contract) => contract.container_image === containerImage); }; + +export const mapContractDtoV0ToContractEntityFields = (params: { + catalogId: string; + contractDto: CatalogContractDtoV0; + logoUri: string | null; +}): CatalogContractEntityFields => { + const { catalogId, contractDto, logoUri } = params; + return { + catalog_id: catalogId, + contract_id: `${contractDto.slug}-${contractDto.container_version}`, + title: contractDto.title, + slug: contractDto.slug, + description: contractDto.description, + short_description: contractDto.short_description, + logo_uri: logoUri ?? undefined, + use_cases: contractDto.use_cases, + verified: contractDto.verified, + last_verified_date: contractDto.last_verified_date ?? undefined, + playbook_supported: contractDto.playbook_supported, + max_confidence_level: contractDto.max_confidence_level, + support_version: contractDto.support_version ?? undefined, + subscription_link: contractDto.subscription_link ?? undefined, + source_code: contractDto.source_code ?? undefined, + manager_supported: contractDto.manager_supported, + version: contractDto.container_version, + image: contractDto.container_image, + connector_type: contractDto.container_type, + config_schema: contractDto.config_schema, + license_type: contractDto.license_type ?? undefined, + solution_categories: contractDto.solution_categories ?? undefined, + contact: contractDto.contact ?? undefined, + }; +}; + +export const mapContractEntityFieldsToGraphqlCatalogContract = (contract: CatalogContractEntityFields): GraphqlCatalogContract => { + return { + title: contract.title, + slug: contract.slug, + description: contract.description, + short_description: contract.short_description, + logo: contract.logo_uri ?? null, + use_cases: contract.use_cases ?? [], + verified: contract.verified, + last_verified_date: contract.last_verified_date ?? '', + playbook_supported: contract.playbook_supported, + max_confidence_level: contract.max_confidence_level, + support_version: contract.support_version ?? null, + subscription_link: contract.subscription_link ?? null, + source_code: contract.source_code ?? '', + manager_supported: contract.manager_supported, + container_version: contract.version, + container_image: contract.image, + container_type: contract.connector_type, + config_schema: contract.config_schema, + license_type: contract.license_type ?? null, + solution_categories: contract.solution_categories ?? [], + contact: contract.contact ?? null, + }; +}; diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-http.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-http.ts new file mode 100644 index 000000000000..61bdc6756c3c --- /dev/null +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-http.ts @@ -0,0 +1,46 @@ +import type { RequestHandler } from 'express'; +import { createAuthenticatedContext } from '../../http/httpAuthenticatedContext'; +import { downloadFile } from '../../database/raw-file-storage'; +import { logApp } from '../../config/conf'; +import { setCookieError } from '../../http/httpUtils'; +import { CATALOG_CONTRACT_LOGOS_DIR, getMimeTypeFromImageExtension } from './catalog-logo-storage'; + +export const CATALOG_LOGO_VIEW_PATH = '/catalog/logo/*file'; + +export const handleCatalogLogoViewRequest: RequestHandler = async (req, res) => { + try { + const context = await createAuthenticatedContext(req, res, 'catalog_logo_view'); + if (!context.user) { + res.sendStatus(403); + return; + } + const file = req.params.file; + if (typeof file !== 'string') { + throw new Error('Invalid URL format'); + } + const s3Key = `${CATALOG_CONTRACT_LOGOS_DIR}/${file}`; + const stream = await downloadFile(s3Key); + if (!stream) { + const error = new Error('Catalog logo not found'); + logApp.error('Failed to download catalog logo', { cause: error }); + res.status(404).send({ status: 'error', error: error.message }); + return; + } + const extension = file.substring(file.lastIndexOf('.')); + const mimeType = getMimeTypeFromImageExtension(extension); + res.set({ 'Content-Security-Policy': 'sandbox' }); + res.set('Cache-Control', 'private, no-cache, no-store, must-revalidate'); + res.set({ Pragma: 'no-cache' }); + if (mimeType) { + res.set('Content-type', mimeType); + } else { + logApp.warn('Catalog logo: unable to deduce mimeType from extension', { file, extension }); + } + stream.pipe(res); + } catch (exception) { + const error = exception instanceof Error ? exception : new Error('Unknown error'); + setCookieError(res, error.message); + logApp.error('Error viewing catalog logo', { cause: error }); + res.status(503).send({ status: 'error', error: error.message }); + } +}; diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-logo-storage.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-logo-storage.ts new file mode 100644 index 000000000000..ca19d777bab3 --- /dev/null +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-logo-storage.ts @@ -0,0 +1,169 @@ +import crypto from 'crypto'; +import { rawListObjects, rawUpload } from '../../database/raw-file-storage'; +import { parseDataUrl } from '../../utils/data-url'; +import type { CatalogContractDtoV0 } from './catalog-types'; +import { CATALOG_LOGO_VIEW_PATH } from './catalog-http'; + +export const CATALOG_CONTRACT_LOGOS_DIR = 'catalog-logos'; + +export const listCatalogContractLogos = async () => { + const result = await rawListObjects(CATALOG_CONTRACT_LOGOS_DIR, false); + if (!result.Contents) { + return new Set(); + } + return new Set(result.Contents.map((object) => { + return object?.Key?.substring(CATALOG_CONTRACT_LOGOS_DIR.length + 1) ?? ''; + }).filter(Boolean)); +}; + +// https://developer.mozilla.org/en-US/docs/Web/HTTP/Guides/MIME_types/Common_types +// List of image mime types commonly supported by browsers +const IMAGE_MIME_TYPES_TO_EXTENSION = { + 'image/apng': '.apng', + 'image/avif': '.avif', + 'image/bmp': '.bmp', + 'image/gif': '.gif', + 'image/jpeg': '.jpg', + 'image/png': '.png', + 'image/svg+xml': '.svg', + 'image/tiff': '.tiff', + 'image/webp': '.webp', +}; + +const EXTENSION_TO_MIME_TYPES = Object.entries(IMAGE_MIME_TYPES_TO_EXTENSION).reduce((acc, [mimeType, extension]) => { + acc[extension] = mimeType; + return acc; +}, {} as Record); + +const isImageMimeType = (mimeType: string): mimeType is keyof typeof IMAGE_MIME_TYPES_TO_EXTENSION => { + return Object.keys(IMAGE_MIME_TYPES_TO_EXTENSION).includes(mimeType); +}; + +const getExtensionFromImageMimeType = (mimeType: keyof typeof IMAGE_MIME_TYPES_TO_EXTENSION) => { + return IMAGE_MIME_TYPES_TO_EXTENSION[mimeType]; +}; + +export const getMimeTypeFromImageExtension = (extension: string) => { + return EXTENSION_TO_MIME_TYPES[extension]; +}; + +const CATALOG_CONTRACT_LOGO_MAX_SIZE_BYTES = 20 * 1024 * 1024; // 20MB + +const isLikelyBase64 = (payload: string) => { + const sanitizedPayload = payload.replace(/\s+/g, ''); + if (sanitizedPayload.length === 0 || sanitizedPayload.length % 4 !== 0) { + return false; + } + return /^[A-Za-z0-9+/]*={0,2}$/.test(sanitizedPayload); +}; + +const computeLogoHash = async (logoContent: Buffer) => { + const hash = crypto.createHash('md5').update(logoContent).digest('hex'); + return hash; +}; + +export const storeCatalogContractLogo = async ( + contractDto: CatalogContractDtoV0, + existingLogos: Set, +) => { + const { logo } = contractDto; + if (!logo) { + return { + result: 'no-logo' as const, + logoUri: null, + }; + } + if (!logo.startsWith('data:')) { + const error = new Error('Unsupported logo data format: not a data URL'); + return { + result: 'failed' as const, + logoUri: null, + error, + }; + } + const { mimeType, base64Encoded, data } = parseDataUrl(logo); + if (!isImageMimeType(mimeType)) { + const error = new Error('Unsupported logo mime type', { + cause: { + mimeType, + }, + }); + return { + result: 'failed' as const, + logoUri: null, + error, + }; + } + let decodedData: Buffer; + if (base64Encoded) { + if (!isLikelyBase64(data)) { + const error = new Error('Unsupported logo data format: invalid base64 payload'); + return { + result: 'failed' as const, + logoUri: null, + error, + }; + } + decodedData = Buffer.from(data, 'base64'); + } else { + try { + decodedData = Buffer.from(decodeURIComponent(data), 'utf8'); + } catch (err: unknown) { + const error = new Error('Unsupported logo data format: invalid URL-encoded payload', { + cause: err, + }); + return { + result: 'failed' as const, + logoUri: null, + error, + }; + } + } + if (decodedData.byteLength === 0) { + const error = new Error('Unexpected zero-length logo data'); + return { + result: 'failed' as const, + logoUri: null, + error, + }; + } + if (decodedData.byteLength > CATALOG_CONTRACT_LOGO_MAX_SIZE_BYTES) { + const error = new Error(`Unsupported logo data format: image exceeds max size (${CATALOG_CONTRACT_LOGO_MAX_SIZE_BYTES} bytes)`); + return { + result: 'failed' as const, + logoUri: null, + error, + }; + } + const hash = await computeLogoHash(decodedData); + const extension = getExtensionFromImageMimeType(mimeType); + const filename = `${hash}${extension}`; + const s3Key = `${CATALOG_CONTRACT_LOGOS_DIR}/${filename}`; + const logoUri = CATALOG_LOGO_VIEW_PATH.replace('*file', s3Key); + if (existingLogos.has(filename)) { + return { + result: 'success' as const, + existed: true, + logoUri, + filename, + }; + } + try { + await rawUpload(s3Key, decodedData); + } catch (err: unknown) { + const error = new Error('Failed to upload logo content to file storage', { + cause: err, + }); + return { + result: 'failed' as const, + logoUri: null, + error, + }; + } + return { + result: 'success' as const, + existed: false, + logoUri, + filename, + }; +}; diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-types.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-types.ts index c2eaee5eda3f..88e74741abbf 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-types.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-types.ts @@ -1,3 +1,6 @@ +import type { BasicStoreEntity, StoreEntity } from '../../types/store'; + +// region DTO types export type IngestionConnectorType = 'INTERNAL_ENRICHMENT' | 'EXTERNAL_IMPORT' | 'INTERNAL_EXPORT_FILE' | 'INTERNAL_IMPORT_FILE'; type TypeMap = { @@ -15,19 +18,19 @@ type TypedProperty = { format?: string; }; -export interface CatalogContract { +export interface CatalogContractDtoV0 { title: string; slug: string; description: string; short_description: string; - logo: string; + logo: string | null; use_cases: string[]; verified: boolean; last_verified_date: string; playbook_supported: boolean; max_confidence_level: number; - support_version: string; - subscription_link: string; + support_version: string | null; + subscription_link: string | null; source_code: string; manager_supported: boolean; container_version: string; @@ -43,21 +46,31 @@ export interface CatalogContract { required: string[]; additionalProperties: boolean; }; + license_type: 'Free' | 'Commercial' | null; + solution_categories: string[]; + contact: string | null; } -export interface CatalogDefinition { +export interface CatalogDtoV0 { id: string; name: string; description: string; - contracts: Array; + contracts: Array; } +// endregion + +// region Domain types +export type CatalogContract = CatalogContractDtoV0; +export type CatalogDefinition = CatalogDtoV0; +// endregion export interface CatalogType { definition: CatalogDefinition; graphql: GraphqlCatalog; } -// region Database types +// region Api types +export type GraphqlCatalogContract = CatalogContractDtoV0; export interface GraphqlCatalog { id: string; entity_type: string; @@ -65,5 +78,67 @@ export interface GraphqlCatalog { parent_types: string[]; name: string; description: string; - contracts: string[]; + contracts: string[]; // JSON.Stringified GraphqlCatalogContract items +} +// endregion + +// region Database types + +export const ENTITY_TYPE_CATALOG_CONTRACT = 'CatalogContract'; +export const ENTITY_TYPE_CATALOG_MANIFEST = 'CatalogManifest'; + +/** + * Fields specific to the `CatalogContract` entity. + * Also reused to embed the contract in `Connector` entities. + */ +export interface CatalogContractEntityFields { + catalog_id: string; + contract_id: string; + title: string; + slug: string; + description: string; + short_description: string; + logo_uri?: string; + use_cases?: string[]; + verified: boolean; + last_verified_date?: string; + playbook_supported: boolean; + max_confidence_level: number; + support_version?: string; + subscription_link?: string; + source_code?: string; + manager_supported: boolean; + version: string; + image: string; + connector_type: IngestionConnectorType; + config_schema: { + $schema: string; + $id: string; + type: string; + properties: { + [key: string]: TypedProperty; + }; + required: string[]; + additionalProperties: boolean; + }; + license_type?: 'Free' | 'Commercial'; + solution_categories: string[]; + contact?: string; +}; + +export interface BasicStoreEntityCatalogContract extends BasicStoreEntity, CatalogContractEntityFields {} + +export interface StoreEntityCatalogContract extends StoreEntity, CatalogContractEntityFields {} + +interface CatalogManifestEntityFields { + source_uri: string; + catalog_id: string; + revision: string; + manifest_version?: string; + version?: string; } + +export interface BasicStoreEntityCatalogManifest extends BasicStoreEntity, CatalogManifestEntityFields {} + +export interface StoreEntityCatalogManifest extends StoreEntity, CatalogManifestEntityFields {} +// endregion diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog.graphql b/opencti-platform/opencti-graphql/src/modules/catalog/catalog.graphql index c405a4467dc7..54595f010340 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog.graphql +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog.graphql @@ -8,12 +8,12 @@ type Catalog implements InternalObject & BasicObject { # Catalog name: String! description: String! - contracts: [String!]! + contracts: [String!]! # JSON.Stringified GraphqlCatalogContract items } type ExtendedContract { catalog_id: String! - contract: String! + contract: String! # JSON.Stringified GraphqlCatalogContract items } # Ordering diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog.ts new file mode 100644 index 000000000000..d2ec66e509a8 --- /dev/null +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog.ts @@ -0,0 +1,99 @@ +import { type ModuleDefinition, registerDefinition } from '../../schema/module'; +import { ABSTRACT_INTERNAL_OBJECT } from '../../schema/general'; +import { + type BasicStoreEntityCatalogContract, + type BasicStoreEntityCatalogManifest, + ENTITY_TYPE_CATALOG_MANIFEST, + ENTITY_TYPE_CATALOG_CONTRACT, + type StoreEntityCatalogContract, + type StoreEntityCatalogManifest, +} from './catalog-types'; +import type { AttributeDefinition } from '../../schema/attribute-definition'; + +export const CATALOG_CONTRACT_MAPPINGS: AttributeDefinition[] = [ + { name: 'catalog_id', label: 'Catalog ID', type: 'string', format: 'short', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: true }, + { name: 'contract_id', label: 'Catalog Contract ID', type: 'string', format: 'short', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: true }, + { name: 'title', label: 'Title', type: 'string', format: 'short', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: true }, + { name: 'slug', label: 'Slug', type: 'string', format: 'short', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: true }, + { name: 'description', label: 'Description', type: 'string', format: 'text', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: false }, + { name: 'short_description', label: 'Short description', type: 'string', format: 'text', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: false }, + { name: 'logo_uri', label: 'Logo URI in file storage', type: 'string', format: 'short', mandatoryType: 'no', editDefault: false, multiple: false, upsert: false, isFilterable: true }, + { name: 'use_cases', label: 'Use cases', type: 'string', format: 'short', mandatoryType: 'external', editDefault: false, multiple: true, upsert: false, isFilterable: true }, + { name: 'verified', label: 'Verified', type: 'boolean', mandatoryType: 'no', editDefault: false, multiple: false, upsert: false, isFilterable: true }, + { name: 'last_verified_date', label: 'Last verified date', type: 'string', format: 'short', mandatoryType: 'no', editDefault: false, multiple: false, upsert: false, isFilterable: false }, + { name: 'playbook_supported', label: 'Playbook supported', type: 'boolean', mandatoryType: 'no', editDefault: false, multiple: false, upsert: false, isFilterable: true }, + { name: 'max_confidence_level', label: 'Max confidence level', type: 'numeric', precision: 'integer', mandatoryType: 'no', editDefault: false, multiple: false, upsert: false, isFilterable: false }, + { name: 'support_version', label: 'Support version', type: 'string', format: 'short', mandatoryType: 'no', editDefault: false, multiple: false, upsert: false, isFilterable: false }, + { name: 'subscription_link', label: 'Subscription link', type: 'string', format: 'short', mandatoryType: 'no', editDefault: false, multiple: false, upsert: false, isFilterable: false }, + { name: 'source_code', label: 'Source code', type: 'string', format: 'short', mandatoryType: 'no', editDefault: false, multiple: false, upsert: false, isFilterable: false }, + { name: 'manager_supported', label: 'Manager supported', type: 'boolean', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: true }, + { name: 'version', label: 'Version', type: 'string', format: 'short', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: true }, + { name: 'image', label: 'Image', type: 'string', format: 'short', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: true }, + { name: 'connector_type', label: 'Connector type', type: 'string', format: 'short', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: true }, + { name: 'config_schema', label: 'Configuration schema', type: 'object', format: 'raw', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: false }, + { name: 'license_type', label: 'License type', type: 'string', format: 'enum', values: ['Free', 'Commercial'], mandatoryType: 'no', editDefault: false, multiple: false, upsert: false, isFilterable: true }, + { name: 'solution_categories', label: 'Solution categories', type: 'string', format: 'short', mandatoryType: 'no', editDefault: false, multiple: true, upsert: false, isFilterable: true }, + { name: 'contact', label: 'Contact', type: 'string', format: 'short', mandatoryType: 'no', editDefault: false, multiple: false, upsert: false, isFilterable: true }, +]; + +// CatalogContract = one persisted entry per (slug, version). +// All connector metadata lives here since any field can vary between versions. +const CATALOG_CONTRACT_DEFINITION: ModuleDefinition = { + type: { + id: 'catalogContracts', + name: ENTITY_TYPE_CATALOG_CONTRACT, + category: ABSTRACT_INTERNAL_OBJECT, + aliased: false, + }, + identifier: { + definition: {}, + resolvers: {}, + }, + attributes: CATALOG_CONTRACT_MAPPINGS, + relations: [], + // Irrelevant as no corresponding Stix object. + // To remove once entity/module definition helper is reworked. + representative: () => { + throw new Error('Should not be called'); + }, + converter_2_1: () => { + throw new Error('Should not be called'); + }, +}; + +registerDefinition(CATALOG_CONTRACT_DEFINITION); + +const CATALOG_MANIFEST_DEFINITION: ModuleDefinition = { + type: { + id: 'catalogManifests', + name: ENTITY_TYPE_CATALOG_MANIFEST, + category: ABSTRACT_INTERNAL_OBJECT, + aliased: false, + }, + identifier: { + definition: { + [ENTITY_TYPE_CATALOG_MANIFEST]: [{ src: 'catalog_id' }], + }, + resolvers: {}, + }, + attributes: [ + { name: 'source_uri', label: 'Source URI', type: 'string', format: 'short', mandatoryType: 'internal', editDefault: false, multiple: false, upsert: true, isFilterable: true }, + { name: 'catalog_id', label: 'Catalog ID', type: 'string', format: 'short', mandatoryType: 'internal', editDefault: false, multiple: false, upsert: true, isFilterable: true }, + { name: 'revision', label: 'Revision', type: 'string', format: 'short', mandatoryType: 'internal', editDefault: false, multiple: false, upsert: true, isFilterable: true }, + { name: 'manifest_version', label: 'Manifest version', type: 'string', format: 'short', mandatoryType: 'no', editDefault: false, multiple: false, upsert: true, isFilterable: false }, + { name: 'version', label: 'Product version', type: 'string', format: 'short', mandatoryType: 'no', editDefault: false, multiple: false, upsert: true, isFilterable: false }, + ], + relations: [], + // Irrelevant as no corresponding Stix object. + // To remove once entity/module definition helper is reworked. + representative: () => { + throw new Error('Should not be called'); + }, + converter_2_1: () => { + throw new Error('Should not be called'); + }, +}; + +registerDefinition(CATALOG_MANIFEST_DEFINITION); + +export type { BasicStoreEntityCatalogContract, BasicStoreEntityCatalogManifest }; diff --git a/opencti-platform/opencti-graphql/src/types/connector.d.ts b/opencti-platform/opencti-graphql/src/types/connector.d.ts index e1f20703e637..c5afb8ec7191 100644 --- a/opencti-platform/opencti-graphql/src/types/connector.d.ts +++ b/opencti-platform/opencti-graphql/src/types/connector.d.ts @@ -1,3 +1,4 @@ +import type { CatalogContractEntityFields } from '../modules/catalog/catalog-types'; import type { BasicStoreEntity, StoreEntity } from './store'; export interface ConnectorInfo { @@ -9,6 +10,8 @@ export interface ConnectorInfo { last_run_datetime: DateTime; } +type ConnectorManagerContract = CatalogContractEntityFields; + export interface BasicStoreEntityConnector extends StoreEntity { active: boolean; auto: boolean; @@ -20,13 +23,15 @@ export interface BasicStoreEntityConnector extends StoreEntity { connector_state: string; connector_state_reset: boolean; connector_trigger_filters: string; - connector_user_id; string; + connector_user_id: string; connector_info: ConnectorInfo; playbook_compatible: boolean; xtm_one_intent: string | null; // region composer (set only on composer-managed connectors) catalog_id?: string; manager_contract_image?: string; + manager_contract?: ConnectorManagerContract; + manager_upgrade_strategy?: string; // endregion } export interface BasicStoreEntityConnectorManager extends BasicStoreEntity { diff --git a/opencti-platform/opencti-graphql/src/utils/data-url.ts b/opencti-platform/opencti-graphql/src/utils/data-url.ts new file mode 100644 index 000000000000..9ba23b1a4643 --- /dev/null +++ b/opencti-platform/opencti-graphql/src/utils/data-url.ts @@ -0,0 +1,32 @@ +export const parseDataUrl = (dataUrl: string) => { + const defaultValue = { + mimeType: '', + base64Encoded: false, + data: '', + }; + + if (!dataUrl.startsWith('data:')) { + return defaultValue; + } + + const metadataAndData = dataUrl.slice(5); + const separatorIndex = metadataAndData.indexOf(','); + if (separatorIndex < 0) { + return defaultValue; + } + + const metadata = metadataAndData.slice(0, separatorIndex); + const data = metadataAndData.slice(separatorIndex + 1); + const metadataParts = metadata + .split(';') + .map((part) => part.trim()) + .filter((part) => part.length > 0); + + const mimeType = metadataParts[0]?.includes('/') ? metadataParts[0].toLowerCase() : ''; + const base64Encoded = metadataParts.some((part) => part.toLowerCase() === 'base64'); + return { + mimeType, + base64Encoded, + data, + }; +}; \ No newline at end of file diff --git a/opencti-platform/opencti-graphql/tests/03-integration/02-resolvers/connector-composer-test.ts b/opencti-platform/opencti-graphql/tests/03-integration/02-resolvers/connector-composer-test.ts index 32f0e7a7a587..f5e4f95b3b8e 100644 --- a/opencti-platform/opencti-graphql/tests/03-integration/02-resolvers/connector-composer-test.ts +++ b/opencti-platform/opencti-graphql/tests/03-integration/02-resolvers/connector-composer-test.ts @@ -98,6 +98,16 @@ const DELETE_CONNECTOR_MUTATION = gql` } `; +const CREATE_CONNECTOR_MUTATION = gql` + mutation RegisterConnector($input: RegisterConnectorInput) { + registerConnector(input: $input) { + id + name + connector_type + } + } +`; + // Queries const CONNECTOR_MANAGER_QUERY = gql` query ConnectorManager($managerId: ID!) { @@ -168,6 +178,65 @@ const GET_CONNECTOR_EXCERPT_QUERY = gql` } `; +const GET_CONNECTOR_MANAGER_CONTRACT_ENDPOINTS_QUERY = gql` + query GetConnectorManagerContractEndpoints($id: String!) { + connector(id: $id) { + id + manager_contract_image + manager_contract_hash + manager_contract_configuration { + key + value + } + manager_contract_definition + manager_contract_excerpt { + title + slug + logo + } + } + } +`; + +const CONNECTOR_MIGRATION_ASSESSMENT_QUERY = gql` + query ConnectorMigrationAssessment($connectorId: ID!, $containerImage: String!, $configuration: [ContractConfigInput!]) { + connectorMigrationAssessment(connectorId: $connectorId, containerImage: $containerImage, configuration: $configuration) { + connector_id + connector_name + contract_image + summary { + total_source_keys + mapped_keys + ignored_keys + missing_mandatory_keys + can_migrate + assessment_date + } + mapped { + key + sourceKey + value + type + required + } + ignored { + key + value + reason + } + missing { + key + type + description + format + enum + default + required + } + } + } +`; + // Variables const ipinfoProperties = { CONNECTOR_LISTEN_PROTOCOL_API_PORT: 0, @@ -1195,6 +1264,7 @@ describe('Connector Composer and Managed Connectors', () => { describe('Managed Connector operations', () => { let managedConnectorId: string; + let standaloneConnectorId: string; it('should fail to add managed connector with invalid image', async () => { const catalogId = catalogHelper.getCatalogId(); @@ -1253,6 +1323,60 @@ describe('Connector Composer and Managed Connectors', () => { expect(managedConnectorAdd.manager_contract_configuration.length).toBeGreaterThan(0); }); + it('should return stable manager contract endpoint output formats', async () => { + const testConnector = catalogHelper.getTestSafeConnector(); + const result = await queryAsAdminWithSuccess({ + query: GET_CONNECTOR_MANAGER_CONTRACT_ENDPOINTS_QUERY, + variables: { id: managedConnectorId }, + }); + + const connectorData = result.data?.connector; + expect(connectorData).toBeDefined(); + expect(typeof connectorData.manager_contract_image).toBe('string'); + expect(connectorData.manager_contract_image).toEqual(`${testConnector.container_image}:${testConnector.container_version}`); + expect(typeof connectorData.manager_contract_hash).toBe('string'); + expect(connectorData.manager_contract_hash.length).toBeGreaterThan(0); + expect(Array.isArray(connectorData.manager_contract_configuration)).toBeTruthy(); + expect(connectorData.manager_contract_configuration.length).toBeGreaterThan(0); + connectorData.manager_contract_configuration.forEach((entry: { key: string; value: string }) => { + expect(typeof entry.key).toBe('string'); + expect(typeof entry.value).toBe('string'); + }); + + const definitionRaw = connectorData.manager_contract_definition; + expect(typeof definitionRaw).toBe('string'); + const definition = JSON.parse(definitionRaw); + expect(definition).toMatchObject({ + title: expect.any(String), + slug: expect.any(String), + description: expect.any(String), + short_description: expect.any(String), + logo: expect.any(String), + use_cases: expect.any(Array), + verified: expect.any(Boolean), + last_verified_date: expect.any(String), + playbook_supported: expect.any(Boolean), + max_confidence_level: expect.any(Number), + support_version: expect.any(String), + subscription_link: expect.any(String), + source_code: expect.any(String), + manager_supported: expect.any(Boolean), + container_version: expect.any(String), + container_image: expect.any(String), + container_type: expect.any(String), + }); + expect(definition.config_schema).toBeDefined(); + expect(definition.config_schema.properties).toBeDefined(); + + const excerpt = connectorData.manager_contract_excerpt; + expect(excerpt).toBeDefined(); + expect(excerpt).toMatchObject({ + title: expect.any(String), + slug: expect.any(String), + logo: expect.any(String), + }); + }); + it('should retrieve the manager contract excerpt for a managed connector', async () => { const result = await queryAsAdminWithSuccess({ query: GET_CONNECTOR_EXCERPT_QUERY, @@ -1264,7 +1388,78 @@ describe('Connector Composer and Managed Connectors', () => { expect(excerpt).toBeDefined(); expect(excerpt.title).toEqual('IPinfo'); expect(excerpt.slug).toEqual('ipinfo'); - expect(excerpt.logo).toMatch(/^data:image\/.+;base64,[A-Za-z0-9+/]+=*$/); + expect(excerpt.logo).toMatch(/^\/storage\/view\/catalog-logos\//); + }); + + it('should return connectorMigrationAssessment with stable output format', async () => { + const testConnector = catalogHelper.getTestSafeConnector(); + const registerResult = await queryAsAdminWithSuccess({ + query: CREATE_CONNECTOR_MUTATION, + variables: { + input: { + id: uuidv4(), + name: 'Assessment standalone connector', + type: testConnector.container_type, + scope: ['IPv4-Addr'], + auto: true, + only_contextual: false, + }, + }, + }); + standaloneConnectorId = registerResult.data?.registerConnector?.id; + createdConnectorIds.add(standaloneConnectorId); + + const assessmentResult = await queryAsAdminWithSuccess({ + query: CONNECTOR_MIGRATION_ASSESSMENT_QUERY, + variables: { + connectorId: standaloneConnectorId, + containerImage: testConnector.container_image, + configuration: catalogHelper.getMinimalConfig(testConnector, { + IPINFO_TOKEN: 'assessment-token', + ...ipinfoProperties, + }), + }, + }); + + const assessment = assessmentResult.data?.connectorMigrationAssessment; + expect(assessment).toBeDefined(); + expect(assessment.connector_id).toEqual(standaloneConnectorId); + expect(assessment.connector_name).toEqual('Assessment standalone connector'); + expect(assessment.contract_image).toEqual(testConnector.container_image); + expect(assessment.summary).toMatchObject({ + total_source_keys: expect.any(Number), + mapped_keys: expect.any(Number), + ignored_keys: expect.any(Number), + missing_mandatory_keys: expect.any(Number), + can_migrate: expect.any(Boolean), + }); + expect(assessment.summary.assessment_date).toBeDefined(); + expect(new Date(assessment.summary.assessment_date).toString()).not.toEqual('Invalid Date'); + expect(Array.isArray(assessment.mapped)).toBeTruthy(); + expect(Array.isArray(assessment.ignored)).toBeTruthy(); + expect(Array.isArray(assessment.missing)).toBeTruthy(); + assessment.mapped.forEach((entry: { key: string; sourceKey: string; value: string; type: string; required: boolean }) => { + expect(entry).toMatchObject({ + key: expect.any(String), + sourceKey: expect.any(String), + value: expect.any(String), + type: expect.any(String), + required: expect.any(Boolean), + }); + }); + assessment.ignored.forEach((entry: { key: string; value: string; reason: string }) => { + expect(entry).toMatchObject({ + key: expect.any(String), + value: expect.any(String), + reason: expect.any(String), + }); + }); + assessment.missing.forEach((entry: { key: string; type: string; description: string; required: boolean }) => { + expect(entry.key).toEqual(expect.any(String)); + expect(entry.type).toEqual(expect.any(String)); + expect(entry.description).toEqual(expect.any(String)); + expect(entry.required).toEqual(expect.any(Boolean)); + }); }); it('should edit managed connector', async () => { From 91c0753f8956c787bc26c2f6c22dfeee10591e34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Tue, 18 Aug 2026 13:03:56 +0200 Subject: [PATCH 03/45] feat: sync catalog to ES & read from ES --- opencti-platform/opencti-graphql/package.json | 2 +- .../src/domain/connector-migration.ts | 35 +- .../opencti-graphql/src/domain/connector.ts | 41 +-- .../src/manager/telemetryManager.ts | 4 +- ...ed-catalog-contract-in-connector-entity.ts | 8 +- .../src/modules/catalog/catalog-domain.ts | 210 ++++------- .../modules/catalog/catalog-logo-storage.ts | 69 +++- .../src/modules/catalog/catalog-manager.ts | 78 ++++ .../src/modules/catalog/catalog-repository.ts | 169 +++++++++ .../src/modules/catalog/catalog-resolver.ts | 10 +- .../src/modules/catalog/catalog-types.ts | 27 +- .../src/modules/catalog/catalog.ts | 10 +- .../catalog/sync/catalog-sync-domain.ts | 335 ++++++++++++++++++ .../sync/catalog-sync-source-gateway.ts | 100 ++++++ .../catalog/sync/catalog-sync-types.ts | 22 ++ .../src/modules/connector/connector-domain.ts | 142 ++++++++ .../modules/connector/connector-repository.ts | 23 ++ .../src/schema/schema-attributes.ts | 4 +- .../src/telemetry/TelemetryMeterManager.ts | 5 +- .../opencti-graphql/src/types/connector.d.ts | 2 + .../opencti-graphql/src/utils/data-url.ts | 2 +- .../catalog-domain/ajvValidatorCache-test.ts | 14 - .../connector-queue-threshold-test.ts | 105 +++--- .../getCatalogsRaceCondition-test.ts | 151 -------- .../validateContractConfigurations-test.ts | 123 +++---- .../01-unit/manager/telemetryTags-test.ts | 17 +- .../02-resolvers/connector-composer-test.ts | 5 - .../02-resolvers/connector-migration-test.ts | 19 +- 28 files changed, 1184 insertions(+), 548 deletions(-) create mode 100644 opencti-platform/opencti-graphql/src/modules/catalog/catalog-manager.ts create mode 100644 opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts create mode 100644 opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts create mode 100644 opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-source-gateway.ts create mode 100644 opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-types.ts create mode 100644 opencti-platform/opencti-graphql/src/modules/connector/connector-domain.ts create mode 100644 opencti-platform/opencti-graphql/src/modules/connector/connector-repository.ts delete mode 100644 opencti-platform/opencti-graphql/tests/01-unit/domain/catalog-domain/getCatalogsRaceCondition-test.ts diff --git a/opencti-platform/opencti-graphql/package.json b/opencti-platform/opencti-graphql/package.json index dc389b1d8698..6fe115887043 100644 --- a/opencti-platform/opencti-graphql/package.json +++ b/opencti-platform/opencti-graphql/package.json @@ -17,7 +17,7 @@ "frontend:stack:analysis": "node ./script/script-stack-analysis front", "lint": "DEBUG=eslint:cli-engine TIMING=1 eslint --quiet --cache --config eslint.config.js src", "build": "run install:python && run build:prod", - "start": "run build:dev && node --enable-source-maps build/back.mjs", + "start": "run build:dev && node --enable-source-maps --inspect build/back.mjs", "start:e2e": "run build:dev && APP__BASE_PATH=/e2e APP__ENABLED_DEV_FEATURES='[\"*\"]' EXPIRATION_SCHEDULER__ENABLED=false REDIS__NAMESPACE=e2e-start RABBITMQ__QUEUE_PREFIX=e2e MINIO__BUCKET_NAME=e2e-start-bucket ELASTICSEARCH__INDEX_PREFIX=e2e-start node build/back.mjs", "start:light": "NODE_ENV=development node --enable-source-maps build/back.mjs", "start:cluster": "NODE_ENV=cluster node --enable-source-maps build/back.mjs", diff --git a/opencti-platform/opencti-graphql/src/domain/connector-migration.ts b/opencti-platform/opencti-graphql/src/domain/connector-migration.ts index 46b2a7051259..29998282320f 100644 --- a/opencti-platform/opencti-graphql/src/domain/connector-migration.ts +++ b/opencti-platform/opencti-graphql/src/domain/connector-migration.ts @@ -7,16 +7,15 @@ import { completeConnector, connector, connectors } from '../database/repository import type { Connector, ConnectorContractConfiguration, ContractConfigInput } from '../generated/graphql'; import { publishUserAction } from '../listener/UserActionListener'; import { addConnectorDeployedCount } from '../manager/telemetryManager'; -import { computeConnectorTargetContract, findContractByContainerImage, mapContractDtoV0ToContractEntityFields } from '../modules/catalog/catalog-domain'; +import { computeConnectorTargetContract, mapCatalogContractToGraphqlCatalogContractWithoutExcludedConfigVars } from '../modules/catalog/catalog-domain'; import { ABSTRACT_INTERNAL_OBJECT } from '../schema/general'; import { ENTITY_TYPE_CONNECTOR, ENTITY_TYPE_CONNECTOR_MANAGER } from '../schema/internalObject'; import type { BasicStoreEntityConnectorManager } from '../types/connector'; import type { AuthContext, AuthUser } from '../types/user'; import { isServiceAccountUser } from '../utils/access'; import { resolveUserByIdFromCache, userEditField } from './user'; -import type { CatalogContractDtoV0 } from '../modules/catalog/catalog-types'; import { now } from '../utils/format'; -import { listCatalogContractLogos, storeCatalogContractLogo } from '../modules/catalog/catalog-logo-storage'; +import { findLatestCompatibleCatalogContractByImageName } from '../modules/catalog/catalog-repository'; type ConfigInput = { key: string; @@ -144,17 +143,12 @@ export const assessConnectorMigration = async (context: AuthContext, user: AuthU throw FunctionalError('Connector is already managed', { id: connectorId }); } - const contractData = await findContractByContainerImage(context, user, containerImage); + const contractData = await findLatestCompatibleCatalogContractByImageName(context, user, containerImage); if (!contractData) { throw FunctionalError('Contract not found', { container_image: containerImage }); } - let contract: CatalogContractDtoV0; - try { - contract = JSON.parse(contractData.contract) as CatalogContractDtoV0; - } catch { - throw FunctionalError('Cannot parse contract found'); - } + const contract = mapCatalogContractToGraphqlCatalogContractWithoutExcludedConfigVars(contractData); // Check type are correct if (existingConnector.connector_type !== contract.container_type) { @@ -257,17 +251,12 @@ export const migrateConnectorToManaged = async ( convertUserToServiceAccount: boolean = true, resetConnectorState: boolean = false, ) => { - const contractData = await findContractByContainerImage(context, user, containerImage); + const contractData = await findLatestCompatibleCatalogContractByImageName(context, user, containerImage); if (!contractData) { throw FunctionalError('Contract not found', { container_image: containerImage }); } - let contract; - try { - contract = JSON.parse(contractData.contract); - } catch { - throw FunctionalError('Cannot parse contract'); - } + const contract = mapCatalogContractToGraphqlCatalogContractWithoutExcludedConfigVars(contractData); if (!contract.manager_supported) { throw FunctionalError('Connector is not managed by composer'); @@ -310,7 +299,7 @@ export const migrateConnectorToManaged = async ( ); const invalidKeys: string[] = []; - userConfig.forEach((value: string, key: string) => { + userConfig.forEach((_value: string, key: string) => { if (!schemaKeysUpper.has(key)) { invalidKeys.push(key); } @@ -364,19 +353,11 @@ export const migrateConnectorToManaged = async ( ); } - const existingLogos = await listCatalogContractLogos(); - const logoStorageResult = await storeCatalogContractLogo(contract, existingLogos); - const managerContract = mapContractDtoV0ToContractEntityFields({ - catalogId: contractData.catalog_id, - contractDto: contract, - logoUri: logoStorageResult.logoUri, - }); - const managedConnectorData: any = { title: existingConnector.name, catalog_id: contractData.catalog_id, manager_contract_image: contract.container_image, - manager_contract: managerContract, + manager_contract: contractData, manager_contract_configuration: filteredConfigurations, manager_requested_status: 'stopped', }; diff --git a/opencti-platform/opencti-graphql/src/domain/connector.ts b/opencti-platform/opencti-graphql/src/domain/connector.ts index 61f6e4c2696d..be363c1117d8 100644 --- a/opencti-platform/opencti-graphql/src/domain/connector.ts +++ b/opencti-platform/opencti-graphql/src/domain/connector.ts @@ -55,7 +55,7 @@ import { isCompatibleVersionWithMinimal } from '../utils/version'; import { extractEntityRepresentativeName } from '../database/entity-representative'; import type { BasicStoreCommon, StoreEntity } from '../types/store'; import { addConnectorDeployedCount, addWorkbenchDraftConvertionCount, addWorkbenchValidationCount } from '../manager/telemetryManager'; -import { computeConnectorTargetContract, getSupportedContractsByImage, mapContractDtoV0ToContractEntityFields } from '../modules/catalog/catalog-domain'; +import { computeConnectorTargetContract } from '../modules/catalog/catalog-domain'; import { getEntitiesMapFromCache } from '../database/cache'; import { createOnTheFlyUser } from '../modules/user/user-domain'; @@ -67,7 +67,7 @@ import { extractContentFrom } from '../utils/fileToContent'; import type { FileHandle } from 'fs/promises'; import { encryptSynchronizerCredential } from './connector-sync-crypto'; import { verifyIngestionUri } from '../modules/ingestion/ingestion-common'; -import { listCatalogContractLogos, storeCatalogContractLogo } from '../modules/catalog/catalog-logo-storage'; +import { findLatestCompatibleCatalogContractByImageName } from '../modules/catalog/catalog-repository'; const MINIMAL_SYNCHRONIZER_COMPATIBLE_VERSION = '6.9.6'; // Sanitize name for K8s/Docker @@ -90,26 +90,6 @@ const sanitizeContainerName = (label: string): string => { return sanitized; }; -const computeManagerContractSnapshot = async ( - catalogId: string, - contractDto: any, -) => { - const existingLogos = await listCatalogContractLogos(); - const logoStorageResult = await storeCatalogContractLogo(contractDto, existingLogos); - if (logoStorageResult.result === 'failed') { - logApp.warn('[CONNECTOR] Failed to store manager contract logo', { - catalogId, - slug: contractDto.slug, - cause: logoStorageResult.error?.message, - }); - } - return mapContractDtoV0ToContractEntityFields({ - catalogId, - contractDto, - logoUri: logoStorageResult.logoUri, - }); -}; - // region connectors export const connectorForWork = async (context: AuthContext, user: AuthUser, id: string) => { const work = await elLoadById(context, user, id, { type: ENTITY_TYPE_WORK, indices: READ_INDEX_HISTORY }); @@ -223,13 +203,12 @@ export const managedConnectorEdit = async ( user: AuthUser, input: EditManagedConnectorInput, ) => { - const conn: any = await storeLoadById(context, user, input.id, ENTITY_TYPE_CONNECTOR); + const conn = await storeLoadById(context, user, input.id, ENTITY_TYPE_CONNECTOR); if (isEmptyField(conn)) { throw UnsupportedError('Connector not found', { id: input.id }); } - const contractsMap = await getSupportedContractsByImage(); - const targetContract: any = contractsMap.get(conn.manager_contract_image); - if (isEmptyField(targetContract)) { + const targetContract = conn.manager_contract; + if (!targetContract) { throw UnsupportedError('Target contract not found'); } const connectorManagers = await fullEntitiesList(context, user, [ENTITY_TYPE_CONNECTOR_MANAGER]); @@ -246,7 +225,7 @@ export const managedConnectorEdit = async ( const patch: any = { name: input.name, title: input.title, - connector_type: targetContract.container_type, + connector_type: targetContract.connector_type, connector_user_id: input.connector_user_id, manager_contract_configuration: contractConfigurations, }; @@ -279,8 +258,7 @@ export const managedConnectorAdd = async ( input: AddManagedConnectorInput, ) => { // Get contract - const contractsMap = await getSupportedContractsByImage(); - const targetContract: any = contractsMap.get(input.manager_contract_image); + const targetContract = await findLatestCompatibleCatalogContractByImageName(context, user, input.manager_contract_image); if (isEmptyField(targetContract)) { throw UnsupportedError('Target contract not found'); } @@ -324,16 +302,15 @@ export const managedConnectorAdd = async ( } // Create connector - const managerContract = await computeManagerContractSnapshot(input.catalog_id, targetContract); const connectorToCreate: any = { title: input.name, name: sanitizedName, - connector_type: targetContract.container_type, + connector_type: targetContract.connector_type, catalog_id: input.catalog_id, connector_user_id: connectorUser.id, manager_contract_image: input.manager_contract_image, manager_contract_configuration: contractConfigurations, - manager_contract: managerContract, + manager_contract: targetContract, manager_requested_status: 'stopped', connector_state_timestamp: now(), built_in: false, diff --git a/opencti-platform/opencti-graphql/src/manager/telemetryManager.ts b/opencti-platform/opencti-graphql/src/manager/telemetryManager.ts index 33f04de1ba5f..a93f2de1cafb 100644 --- a/opencti-platform/opencti-graphql/src/manager/telemetryManager.ts +++ b/opencti-platform/opencti-graphql/src/manager/telemetryManager.ts @@ -57,7 +57,6 @@ import { ENTITY_TYPE_INGESTION_TAXII_COLLECTION, } from '../modules/ingestion/ingestion-types'; import { ENTITY_TYPE_MANAGER_CONFIGURATION } from '../modules/managerConfiguration/managerConfiguration-types'; -import { getSupportedContractsByImage } from '../modules/catalog/catalog-domain'; import { FilterMode } from '../generated/graphql'; import { redisClearTelemetry, redisGetTelemetry, redisSetTelemetryAdd } from '../database/redis'; import { countOffloadedStreamEvents, rawFetchStreamInfo } from '../database/redis-stream'; @@ -496,8 +495,7 @@ export const fetchTelemetryData = async (manager: TelemetryMeterManager) => { // composer-managed connectors resolve to the catalog contract slug // through their stored container image; manually registered connectors // fall back to their registered name, flagged managed=false. - const contractsByImage = await getSupportedContractsByImage(); - manager.setActiveConnectorsByIdentity(computeActiveConnectorsByIdentity(activeConnectors, contractsByImage)); + manager.setActiveConnectorsByIdentity(computeActiveConnectorsByIdentity(activeConnectors)); // endregion // region Roles with draft capability information diff --git a/opencti-platform/opencti-graphql/src/migrations/1786264797376-embed-catalog-contract-in-connector-entity.ts b/opencti-platform/opencti-graphql/src/migrations/1786264797376-embed-catalog-contract-in-connector-entity.ts index 3dae5e3ae9ef..02c45f915d3f 100644 --- a/opencti-platform/opencti-graphql/src/migrations/1786264797376-embed-catalog-contract-in-connector-entity.ts +++ b/opencti-platform/opencti-graphql/src/migrations/1786264797376-embed-catalog-contract-in-connector-entity.ts @@ -10,6 +10,8 @@ import openCtiManifest from '../__generated__/opencti-manifest.json'; import type { CatalogContract, CatalogDefinition } from '../modules/catalog/catalog-types'; import { mapContractDtoV0ToContractEntityFields } from '../modules/catalog/catalog-domain'; import { listCatalogContractLogos, storeCatalogContractLogo } from '../modules/catalog/catalog-logo-storage'; +import { computeContractContentHash } from '../modules/catalog/sync/catalog-sync-domain'; +import { mapCatalogContractDtoToCatalogContractSyncSource } from '../modules/catalog/sync/catalog-sync-source-gateway'; const message = '[MIGRATION] managed connectors contract snapshot'; @@ -17,7 +19,7 @@ export const up = async (next: (error?: Error) => void) => { logMigration.info(`${message} > started`); const context = executionContext('migration'); - const filigranCatalog = openCtiManifest as CatalogDefinition; + const filigranCatalog = openCtiManifest as unknown as CatalogDefinition; // Load all managed connectors which have their contract definition in the embedded catalog manifest const managedConnectorsArgs = { @@ -52,9 +54,13 @@ export const up = async (next: (error?: Error) => void) => { storedLogos.add(result.filename); } } + const contractContentHash = computeContractContentHash( + mapCatalogContractDtoToCatalogContractSyncSource(contractDto), + ); patch.manager_contract = mapContractDtoV0ToContractEntityFields({ catalogId: filigranCatalog.id, contractDto, + contractContentHash, logoUri: result.logoUri, }); } diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-domain.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-domain.ts index 3e8546f96fa5..09d02638dd34 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-domain.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-domain.ts @@ -3,21 +3,21 @@ import addFormats from 'ajv-formats'; import crypto from 'crypto'; import type { AuthContext, AuthUser } from '../../types/user'; import { + type BasicStoreEntityCatalogContract, + type BasicStoreEntityCatalogManifest, type CatalogContract, type CatalogContractDtoV0, type CatalogContractEntityFields, - type CatalogDefinition, - type CatalogType, + type GraphqlCatalog, type GraphqlCatalogContract, } from './catalog-types'; import { isEmptyField } from '../../database/utils'; import { UnsupportedError } from '../../config/errors'; import { idGenFromData } from '../../schema/identifier'; -import filigranCatalog from '../../__generated__/opencti-manifest.json'; -import conf, { logApp } from '../../config/conf'; +import { logApp } from '../../config/conf'; import type { ConnectorContractConfiguration, ContractConfigInput } from '../../generated/graphql'; -import { readFile } from 'node:fs/promises'; import type { ValidateFunction } from 'ajv'; +import { findCatalogContractsByCatalogId, findCatalogManifestByCatalogId, findCatalogs, findLatestCompatibleCatalogContractBySlug } from './catalog-repository'; const validatorCache = new Map(); @@ -26,7 +26,7 @@ const validatorCache = new Map(); * The cacheKey must accurately reflect the exact shape of the schema (properties + required) * to avoid cache false positives. */ -const getOrCompileValidator = (cacheKey: string, jsonValidation: object): ValidateFunction => { +export const getOrCompileValidator = (cacheKey: string, jsonValidation: object): ValidateFunction => { let validate = validatorCache.get(cacheKey); if (!validate) { validate = ajv.compile(jsonValidation); @@ -35,111 +35,9 @@ const getOrCompileValidator = (cacheKey: string, jsonValidation: object): Valida return validate; }; -const CUSTOM_CATALOGS: string[] = conf.get('app:custom_catalogs') ?? []; const ajv = new Ajv({ coerceTypes: true }); addFormats(ajv, ['password', 'uri', 'duration', 'email', 'date-time', 'date']); -// Cache of catalog to read on disk and parse only once -let catalogMap: Promise> | undefined; -// cache for contracts by image map -let contractsByImageCache: Promise> | undefined; - -// Build catalog map from files -const buildCatalogMap = async (): Promise> => { - const newCatalogMap: Record = {}; - const catalogs = []; - catalogs.push(JSON.stringify(filigranCatalog)); - // Add custom catalogs - for (let index = 0; index < CUSTOM_CATALOGS.length; index += 1) { - const customCatalog = CUSTOM_CATALOGS[index]; - const catalog = await readFile(customCatalog, { encoding: 'utf8', flag: 'r' }); - catalogs.push(catalog); - } - // Prepare catalogs map - for (let index = 0; index < catalogs.length; index += 1) { - const catalogRaw = catalogs[index]; - const catalog = JSON.parse(catalogRaw) as CatalogDefinition; - // Validate each contract - for (let contractIndex = 0; contractIndex < catalog.contracts.length; contractIndex += 1) { - const contract = catalog.contracts[contractIndex]; - if (contract.manager_supported) { - if (!contract.config_schema) { - logApp.warn('A contract has manager_supported=true but is missing config_schema', { contractTitle: contract.title }); - } else { - if (isEmptyField(contract.container_image)) { - throw UnsupportedError('Contract must define container_image field', { contractTitle: contract.title }); - } - if (isEmptyField(contract.container_type)) { - throw UnsupportedError('Contract must define container_type field', { contractTitle: contract.title }); - } - - if (contract.config_schema) { - const jsonValidation = { - type: contract.config_schema.type, - properties: contract.config_schema.properties, - required: contract.config_schema.required, - additionalProperties: contract.config_schema.additionalProperties, - }; - try { - getOrCompileValidator(`catalog-contract:${catalog.id}:${contract.slug}`, jsonValidation); - } catch (err) { - throw UnsupportedError('Contract must be a valid json schema definition', { cause: err }); - } - } - } - } - } - newCatalogMap[catalog.id] = { - definition: catalog, - graphql: { - id: catalog.id, - entity_type: 'Catalog', - parent_types: ['Internal'], - standard_id: idGenFromData('catalog', { id: catalog.id }), - name: catalog.name, - description: catalog.description, - contracts: catalog.contracts.map((c) => { - const finalContract = c; - if (finalContract.manager_supported) { - if (!finalContract.config_schema) { - logApp.warn('A contract has manager_supported=true but is missing config_schema', { contractTitle: finalContract.title }); - } else { - const EXCLUDED_CONFIG_VARS = ['OPENCTI_TOKEN', 'OPENCTI_URL', 'CONNECTOR_TYPE', 'CONNECTOR_RUN_AND_TERMINATE']; - EXCLUDED_CONFIG_VARS.forEach((property) => { - delete finalContract.config_schema.properties[property]; - }); - finalContract.config_schema.required = c.config_schema.required.filter((item) => !EXCLUDED_CONFIG_VARS.includes(item)); - } - } - return JSON.stringify(finalContract); - }), - }, - }; - } - return newCatalogMap; -}; - -// Enable custom catalogs - clears cache and enables live catalog loading -export const resetCatalogs = () => { - catalogMap = undefined; - contractsByImageCache = undefined; - validatorCache.clear(); -}; - -const getCatalogs = async (): Promise> => { - if (!catalogMap) { - // We memoize the Promise immediately (not after it resolves) so that - // all concurrent calls share the same in-flight execution. - catalogMap = buildCatalogMap().catch((err) => { - // On failure, we reset the cache to allow a retry on the next call, - // rather than staying stuck with a rejected Promise cached forever. - catalogMap = undefined; - throw err; - }); - } - return catalogMap; -}; - const aesEncrypt = (text: string, key: Buffer, aesIv: Buffer) => { const cipher = crypto.createCipheriv('aes-256-gcm', key, aesIv); const ciphertext = Buffer.concat([ @@ -326,7 +224,7 @@ const formatValidationErrors = (errors: any[] | null | undefined, contractTitle: export const validateContractConfigurations = ( contractConfigurations: ConnectorContractConfiguration[], - targetContract: CatalogContract, + targetContract: Pick, ) => { const targetConfig = targetContract.config_schema; @@ -393,7 +291,7 @@ export const validateContractConfigurations = ( export const computeConnectorTargetContract = ( configurations: ContractConfigInput[], - targetContract: CatalogContract, + targetContract: Pick, publicKey: string, currentManagerContractConfiguration?: ConnectorContractConfiguration[], ): ConnectorContractConfiguration[] => { @@ -449,67 +347,79 @@ export const computeConnectorTargetContract = ( return contractConfigurations; }; -export const getSupportedContractsByImage = async (): Promise> => { - if (!contractsByImageCache) { - contractsByImageCache = (async () => { - const catalogDefinitions = await getCatalogs(); - const contracts = Object.values(catalogDefinitions).map((catalog) => catalog.definition.contracts).flat(); - return new Map(contracts.map((contract) => [contract.container_image, contract])); - })().catch((err) => { - contractsByImageCache = undefined; - throw err; - }); +const EXCLUDED_CONFIG_VARS = ['OPENCTI_TOKEN', 'OPENCTI_URL', 'CONNECTOR_TYPE', 'CONNECTOR_RUN_AND_TERMINATE']; + +export const mapCatalogContractToGraphqlCatalogContractWithoutExcludedConfigVars = ( + contract: BasicStoreEntityCatalogContract, +): GraphqlCatalogContract => { + const finalContract = mapContractEntityFieldsToGraphqlCatalogContract(contract); + if (finalContract.manager_supported) { + if (!finalContract.config_schema) { + logApp.warn('A contract has manager_supported=true but is missing config_schema', { contractTitle: finalContract.title }); + } else { + EXCLUDED_CONFIG_VARS.forEach((property) => { + delete finalContract.config_schema.properties[property]; + }); + finalContract.config_schema.required = contract.config_schema.required.filter((item) => !EXCLUDED_CONFIG_VARS.includes(item)); + } } - return contractsByImageCache; -}; - -export const findById = async (_context: AuthContext, _user: AuthUser, catalogId: string) => { - const catalogDefinitions = await getCatalogs(); - return catalogDefinitions[catalogId].graphql; + return finalContract; }; -export const findCatalog = async (_context: AuthContext, _user: AuthUser) => { - const catalogDefinitions = await getCatalogs(); - return Object.values(catalogDefinitions).map((catalog) => catalog.graphql); +const mapCatalogToGraphqlCatalog = ( + catalog: BasicStoreEntityCatalogManifest, + contracts: BasicStoreEntityCatalogContract[], +): GraphqlCatalog => { + return { + id: catalog.catalog_id, + name: catalog.name, + description: catalog.description, + entity_type: 'Catalog', + parent_types: ['Internal'], + standard_id: idGenFromData('catalog', { id: catalog.id }), + contracts: contracts.map((c) => + JSON.stringify(mapCatalogContractToGraphqlCatalogContractWithoutExcludedConfigVars(c)), + ), + }; }; -const findContract = async (_context: AuthContext, _user: AuthUser, predicate: (contract: any) => boolean) => { - const catalogDefinitions = await getCatalogs(); - if (!catalogDefinitions) { +export const queryCatalogById = async (context: AuthContext, user: AuthUser, catalogId: string) => { + const catalog = await findCatalogManifestByCatalogId(context, user, catalogId); + if (!catalog) { return null; } - - const catalogs = Object.values(catalogDefinitions).map((catalog) => catalog.graphql); - const foundContract = catalogs - .map((catalog) => { - const contract = catalog.contracts.find((contractStr: string) => { - const parsedContract = JSON.parse(contractStr); - return predicate(parsedContract); - }); - return contract ? { catalog_id: catalog.id, contract } : null; - }) - .find((contract) => contract !== null); - - return foundContract || null; + const contracts = await findCatalogContractsByCatalogId(context, user, catalogId); + return mapCatalogToGraphqlCatalog(catalog, Object.values(contracts)); }; -export const findContractBySlug = (context: AuthContext, user: AuthUser, contractSlug: string) => { - return findContract(context, user, (contract) => contract.slug === contractSlug); +export const queryCatalogs = async (context: AuthContext, user: AuthUser) => { + const catalogs = await findCatalogs(context, user); + const contracts = await Promise.all(catalogs.map((catalog) => findCatalogContractsByCatalogId(context, user, catalog.id))); + return catalogs.map((catalog, idx) => mapCatalogToGraphqlCatalog(catalog, Object.values(contracts[idx]))); }; -export const findContractByContainerImage = (context: AuthContext, user: AuthUser, containerImage: string) => { - return findContract(context, user, (contract) => contract.container_image === containerImage); +export const queryContractBySlug = async (context: AuthContext, user: AuthUser, contractSlug: string) => { + const contract = await findLatestCompatibleCatalogContractBySlug(context, user, contractSlug); + if (!contract) { + return null; + } + return { + catalog_id: contract.catalog_id, + contract: JSON.stringify(mapCatalogContractToGraphqlCatalogContractWithoutExcludedConfigVars(contract)), + }; }; export const mapContractDtoV0ToContractEntityFields = (params: { catalogId: string; contractDto: CatalogContractDtoV0; + contractContentHash: string; logoUri: string | null; }): CatalogContractEntityFields => { - const { catalogId, contractDto, logoUri } = params; + const { catalogId, contractDto, contractContentHash, logoUri } = params; return { catalog_id: catalogId, contract_id: `${contractDto.slug}-${contractDto.container_version}`, + content_hash: contractContentHash, title: contractDto.title, slug: contractDto.slug, description: contractDto.description, diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-logo-storage.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-logo-storage.ts index ca19d777bab3..e5a596983947 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-logo-storage.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-logo-storage.ts @@ -6,6 +6,13 @@ import { CATALOG_LOGO_VIEW_PATH } from './catalog-http'; export const CATALOG_CONTRACT_LOGOS_DIR = 'catalog-logos'; +export type CatalogContractLogoUploadOperation = { + filename: string; + s3Key: string; + body: Buffer; + logoUri: string; +}; + export const listCatalogContractLogos = async () => { const result = await rawListObjects(CATALOG_CONTRACT_LOGOS_DIR, false); if (!result.Contents) { @@ -57,14 +64,46 @@ const isLikelyBase64 = (payload: string) => { return /^[A-Za-z0-9+/]*={0,2}$/.test(sanitizedPayload); }; -const computeLogoHash = async (logoContent: Buffer) => { - const hash = crypto.createHash('md5').update(logoContent).digest('hex'); - return hash; +const computeLogoHash = (logoContent: Buffer) => { + return crypto.createHash('md5').update(logoContent).digest('hex'); }; export const storeCatalogContractLogo = async ( contractDto: CatalogContractDtoV0, existingLogos: Set, +) => { + const operationResult = computeCatalogContractLogoUploadOperation(contractDto, existingLogos); + if (operationResult.result !== 'success' || operationResult.existed) { + return operationResult; + } + if (!operationResult.operation) { + const error = new Error('Missing logo upload operation'); + return { + result: 'failed' as const, + logoUri: null, + error, + }; + } + + const uploadResult = await uploadCatalogContractLogoOperation(operationResult.operation); + if (uploadResult.result === 'failed') { + return { + result: 'failed' as const, + logoUri: null, + error: uploadResult.error, + }; + } + return { + result: 'success' as const, + existed: false, + logoUri: operationResult.logoUri, + filename: operationResult.filename, + }; +}; + +export const computeCatalogContractLogoUploadOperation = ( + contractDto: CatalogContractDtoV0, + existingLogos: Set, ) => { const { logo } = contractDto; if (!logo) { @@ -135,7 +174,7 @@ export const storeCatalogContractLogo = async ( error, }; } - const hash = await computeLogoHash(decodedData); + const hash = computeLogoHash(decodedData); const extension = getExtensionFromImageMimeType(mimeType); const filename = `${hash}${extension}`; const s3Key = `${CATALOG_CONTRACT_LOGOS_DIR}/${filename}`; @@ -148,22 +187,34 @@ export const storeCatalogContractLogo = async ( filename, }; } + const operation: CatalogContractLogoUploadOperation = { + filename, + s3Key, + body: decodedData, + logoUri, + }; + return { + result: 'success' as const, + existed: false, + logoUri, + filename, + operation, + }; +}; + +export const uploadCatalogContractLogoOperation = async (operation: CatalogContractLogoUploadOperation) => { try { - await rawUpload(s3Key, decodedData); + await rawUpload(operation.s3Key, operation.body); } catch (err: unknown) { const error = new Error('Failed to upload logo content to file storage', { cause: err, }); return { result: 'failed' as const, - logoUri: null, error, }; } return { result: 'success' as const, - existed: false, - logoUri, - filename, }; }; diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-manager.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-manager.ts new file mode 100644 index 000000000000..427464f49294 --- /dev/null +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-manager.ts @@ -0,0 +1,78 @@ +import { clearIntervalAsync, setIntervalAsync, type SetIntervalAsyncTimer } from 'set-interval-async/fixed'; +import conf, { booleanConf, logApp } from '../../config/conf'; +import { lockResources } from '../../lock/master-lock'; +import { TYPE_LOCK_ERROR } from '../../config/errors'; +import { executionContext, SYSTEM_USER } from '../../utils/access'; +import { autoUpgradeManagedConnectors } from '../connector/connector-domain'; +import { synchronizeCatalogs } from './sync/catalog-sync-domain'; + +const CATALOG_MANAGER_ENABLED = booleanConf('app:catalog_manager:enabled', true); +const CATALOG_MANAGER_LOCK_KEY = conf.get('app:catalog_manager:lock_key') || 'catalog_manager_lock'; +const CATALOG_MANAGER_INTERVAL = conf.get('app:catalog_manager:interval'); + +let scheduler: SetIntervalAsyncTimer<[]> | undefined; + +export const isCatalogManagerEnabled = () => CATALOG_MANAGER_ENABLED; + +const executeScheduledJob = async () => { + let lock; + try { + lock = await lockResources([CATALOG_MANAGER_LOCK_KEY], { retryCount: 0 }); + const context = executionContext('catalog_manager'); + // Sync catalogs to ES + const syncedCatalogsWithChanges = await synchronizeCatalogs(context, SYSTEM_USER); + // Apply upgrade strategy. + // Should be moved another manager maybe (connectorManager). + // Consider using an event to invert the dependency. + await autoUpgradeManagedConnectors(context, SYSTEM_USER, syncedCatalogsWithChanges); + } catch (error: any) { + if (error?.name === TYPE_LOCK_ERROR) { + logApp.debug('[OPENCTI-MODULE] Catalog manager refresh already running on another node'); + return; + } + throw error; + } finally { + if (lock) { + await lock.unlock(); + } + } +}; + +const triggerExecuteScheduledJob = () => { + if (!CATALOG_MANAGER_ENABLED) { + return; + } + void executeScheduledJob().catch((error) => { + logApp.error('[OPENCTI-MODULE] Catalog manager background refresh failed', { cause: error }); + }); +}; + +const start = async () => { + if (!CATALOG_MANAGER_ENABLED) { + logApp.info('[OPENCTI-MODULE] Catalog manager disabled by configuration'); + return; + } + + triggerExecuteScheduledJob(); + + if (CATALOG_MANAGER_INTERVAL && Number(CATALOG_MANAGER_INTERVAL) > 0) { + scheduler = setIntervalAsync(async () => { + triggerExecuteScheduledJob(); + }, Number(CATALOG_MANAGER_INTERVAL)); + logApp.info(`[OPENCTI-MODULE] Catalog manager scheduled every ${Number(CATALOG_MANAGER_INTERVAL)}ms`); + } else { + logApp.info('[OPENCTI-MODULE] Catalog manager configured for startup-only refresh'); + } +}; + +const shutdown = async () => { + if (scheduler) { + await clearIntervalAsync(scheduler); + scheduler = undefined; + } +}; + +export default { + start, + shutdown, +}; diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts new file mode 100644 index 000000000000..d6303b3e8d64 --- /dev/null +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts @@ -0,0 +1,169 @@ +import { elDeleteInstances, elIndex, elIndexElements, elLoadBy } from '../../database/engine'; +import { fullEntitiesList } from '../../database/middleware-loader'; +import { INDEX_INTERNAL_OBJECTS } from '../../database/utils'; +import { FilterMode, FilterOperator } from '../../generated/graphql'; +import type { BasicStoreBase } from '../../types/store'; +import type { AuthContext, AuthUser } from '../../types/user'; +import { + type BasicStoreEntityCatalogContract, + ENTITY_TYPE_CATALOG_CONTRACT, + type BasicStoreEntityCatalogManifest, + type CatalogContractCreation, + type CatalogContractUpdate, + type CatalogContractDeletion, + type CatalogManifestUpsert, + ENTITY_TYPE_CATALOG_MANIFEST, +} from './catalog-types'; + +/** + * Catalog data accessors & mutators + */ + +export const findCatalogManifestByCatalogId = async ( + context: AuthContext, + user: AuthUser, + catalogId: string, +) => { + const catalog = await elLoadBy(context, user, 'catalog_id', catalogId); + return catalog; +}; + +export const upsertCatalog = async (_context: AuthContext, _user: AuthUser, update: CatalogManifestUpsert) => { + await elIndex(INDEX_INTERNAL_OBJECTS, { + ...update, + entity_type: ENTITY_TYPE_CATALOG_MANIFEST, + }); +}; + +export const findCatalogs = async (context: AuthContext, user: AuthUser, excludedIds?: string[]) => { + const filters = excludedIds?.length ? { + filters: [{ + key: ['catalog_id'], + values: excludedIds, + operator: FilterOperator.NotEq, + }], + filterGroups: [], + mode: FilterMode.And, + } : null; + const catalogs = await fullEntitiesList( + context, + user, + [ENTITY_TYPE_CATALOG_MANIFEST], + { filters }, + ); + return catalogs; +}; + +export const deleteCatalogs = async (context: AuthContext, catalogEntities: BasicStoreEntityCatalogManifest[]) => { + await elDeleteInstances(context, catalogEntities); +}; + +/** + * Catalog contracts data accessors & mutators + */ + +export const findCatalogContractsByCatalogId = async ( + context: AuthContext, + user: AuthUser, + catalogId: string, +) => { + const contracts = await fullEntitiesList( + context, + user, + [ENTITY_TYPE_CATALOG_CONTRACT], + { + filters: { + filters: [{ + key: ['catalog_id'], + values: [catalogId], + }], + filterGroups: [], + mode: FilterMode.And, + }, + }, + ); + // TODO: check if filter with support_version + return contracts.reduce((map, contract) => { + map.set(contract.contract_id, contract); + return map; + }, new Map()); +}; + +export const findLatestCompatibleCatalogContractBySlug = async ( + context: AuthContext, + user: AuthUser, + contractSlug: string, +) => { + const contracts = await fullEntitiesList( + context, + user, + [ENTITY_TYPE_CATALOG_CONTRACT], + { + filters: { + filters: [{ + key: ['slug'], + values: [contractSlug], + }], + filterGroups: [], + mode: FilterMode.And, + }, + }, + ); + // Filter with support_version + // Sort by version DESC + return contracts[0]; +}; + +export const findLatestCompatibleCatalogContractByImageName = async ( + context: AuthContext, + user: AuthUser, + imageName: string, +) => { + const contracts = await fullEntitiesList( + context, + user, + [ENTITY_TYPE_CATALOG_CONTRACT], + { + filters: { + filters: [{ + key: ['image'], + values: [imageName], + }], + filterGroups: [], + mode: FilterMode.And, + }, + }, + ); + // Filter with support_version + // Sort by version DESC + return contracts[0]; +}; + +export const insertCatalogContracts = async ( + context: AuthContext, + user: AuthUser, + contracts: CatalogContractCreation[], +) => { + await elIndexElements(context, user, ENTITY_TYPE_CATALOG_CONTRACT, contracts); +}; + +export const updateCatalogContracts = async ( + context: AuthContext, + user: AuthUser, + updates: CatalogContractUpdate[], +) => { + // We can use bulk `index` as we provide the entire documents + await elIndexElements(context, user, ENTITY_TYPE_CATALOG_CONTRACT, updates); +}; + +export const deleteCatalogContracts = async ( + context: AuthContext, + _user: AuthUser, + deletions: CatalogContractDeletion[], +) => { + const docs = deletions.map((deletion) => ({ + _index: INDEX_INTERNAL_OBJECTS, + _id: deletion.idToDelete, + } as BasicStoreBase)); + await elDeleteInstances(context, docs); +}; diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-resolver.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-resolver.ts index c1b5d15a11c6..e49715ac818e 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-resolver.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-resolver.ts @@ -1,16 +1,16 @@ -import { findCatalog, findById, findContractBySlug } from './catalog-domain'; +import { queryCatalogById, queryCatalogs, queryContractBySlug } from './catalog-domain'; import type { Resolvers } from '../../generated/graphql'; const catalogResolver: Resolvers = { Query: { catalog: (_, { id }, context) => { - return findById(context, context.user, id); + return queryCatalogById(context, context.user, id); }, - catalogs: (_, args, context) => { - return findCatalog(context, context.user); + catalogs: (_, _args, context) => { + return queryCatalogs(context, context.user); }, contract: (_, { slug }, context) => { - return findContractBySlug(context, context.user, slug); + return queryContractBySlug(context, context.user, slug); }, }, }; diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-types.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-types.ts index 88e74741abbf..8491004c3619 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-types.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-types.ts @@ -13,8 +13,8 @@ type TypeMap = { type TypedProperty = { type: K; - default: TypeMap[K]; - description: string; + default?: TypeMap[K]; + description?: string; format?: string; }; @@ -26,7 +26,7 @@ export interface CatalogContractDtoV0 { logo: string | null; use_cases: string[]; verified: boolean; - last_verified_date: string; + last_verified_date: string | null; playbook_supported: boolean; max_confidence_level: number; support_version: string | null; @@ -55,6 +55,7 @@ export interface CatalogDtoV0 { id: string; name: string; description: string; + version: string; contracts: Array; } // endregion @@ -94,6 +95,7 @@ export const ENTITY_TYPE_CATALOG_MANIFEST = 'CatalogManifest'; export interface CatalogContractEntityFields { catalog_id: string; contract_id: string; + content_hash: string; title: string; slug: string; description: string; @@ -131,14 +133,27 @@ export interface BasicStoreEntityCatalogContract extends BasicStoreEntity, Catal export interface StoreEntityCatalogContract extends StoreEntity, CatalogContractEntityFields {} interface CatalogManifestEntityFields { + revision: string; source_uri: string; catalog_id: string; - revision: string; - manifest_version?: string; - version?: string; + name: string; + description: string; + version: string; } export interface BasicStoreEntityCatalogManifest extends BasicStoreEntity, CatalogManifestEntityFields {} export interface StoreEntityCatalogManifest extends StoreEntity, CatalogManifestEntityFields {} + +export interface CatalogContractDeletion { + idToDelete: string; +} + +export type CatalogContractUpdate = Required<{ + [TKey in keyof CatalogContractEntityFields]: CatalogContractEntityFields[TKey] | null; +}>; + +export type CatalogContractCreation = CatalogContractEntityFields; + +export type CatalogManifestUpsert = CatalogManifestEntityFields; // endregion diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog.ts index d2ec66e509a8..b69e82625207 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog.ts @@ -13,6 +13,7 @@ import type { AttributeDefinition } from '../../schema/attribute-definition'; export const CATALOG_CONTRACT_MAPPINGS: AttributeDefinition[] = [ { name: 'catalog_id', label: 'Catalog ID', type: 'string', format: 'short', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: true }, { name: 'contract_id', label: 'Catalog Contract ID', type: 'string', format: 'short', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: true }, + { name: 'content_hash', label: 'Catalog Contract content hash', type: 'string', format: 'short', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: true }, { name: 'title', label: 'Title', type: 'string', format: 'short', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: true }, { name: 'slug', label: 'Slug', type: 'string', format: 'short', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: true }, { name: 'description', label: 'Description', type: 'string', format: 'text', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: false }, @@ -77,11 +78,10 @@ const CATALOG_MANIFEST_DEFINITION: ModuleDefinition { + return createHash('sha256').update(JSON.stringify(rawManifest)).digest('hex'); +}; + +export const computeContractContentHash = (contract: CatalogContractSyncSource) => { + const replacer = (_key: string, value: Record) => + value && typeof value === 'object' && !Array.isArray(value) + ? Object.keys(value).sort().reduce((sorted, k) => { + sorted[k] = value[k]; + return sorted; + }, {} as Record) + : value; + return createHash('sha256').update(JSON.stringify(contract, replacer)).digest('hex'); +}; + +const computeCatalogLogosSyncOps = ( + sourceCatalog: CatalogSyncSource, + currentLogos: Set, +) => { + const plannedLogos = new Set(currentLogos); + const sourceCatalogLogosUris = new Map(); + const uploadOperations: CatalogContractLogoUploadOperation[] = []; + for (const sourceContract of sourceCatalog.contracts) { + const logoOperationResult = computeCatalogContractLogoUploadOperation(sourceContract, plannedLogos); + if (logoOperationResult.result === 'failed') { + throw new Error(`Error while preparing logo for contract ${sourceContract.id}`, { + cause: logoOperationResult.error, + }); + } + if (logoOperationResult.result === 'no-logo') { + continue; + } + sourceCatalogLogosUris.set(sourceContract.id, logoOperationResult.logoUri); + plannedLogos.add(logoOperationResult.filename); + if (!logoOperationResult.existed) { + if (!logoOperationResult.operation) { + throw new Error(`Missing logo upload operation for contract ${sourceContract.id}`); + } + uploadOperations.push(logoOperationResult.operation); + } + } + return { + sourceCatalogLogosUris, + uploadOperations, + }; +}; + +const uploadCatalogContractLogos = async (operations: CatalogContractLogoUploadOperation[]) => { + for (const operation of operations) { + const uploadResult = await uploadCatalogContractLogoOperation(operation); + if (uploadResult.result === 'failed') { + throw uploadResult.error; + } + } +}; + +const computeCatalogSyncOps = (params: { + revision: string; + sourceConfig: CatalogSyncSourceConfig; + sourceCatalog: CatalogSyncSource; + sourceCatalogContractsHashes: Map; + sourceCatalogLogosUris: Map; + currentCatalog: BasicStoreEntityCatalogManifest | undefined; + currentContracts: Map; +}) => { + const sourceContractsIds = new Set(params.sourceCatalogContractsHashes.keys()); + const contractsCreations: CatalogContractCreation[] = []; + const contractsUpdates: CatalogContractUpdate[] = []; + const contractsDeletions: CatalogContractDeletion[] = []; + + params.sourceCatalog.contracts.forEach((sourceContract) => { + const sourceContractHash = params.sourceCatalogContractsHashes.get(sourceContract.id)!; + const currentContract = params.currentContracts.get(sourceContract.id); + if (!currentContract) { + const contractCreation: CatalogContractCreation = { + catalog_id: params.sourceCatalog.id, + contract_id: sourceContract.id, + content_hash: sourceContractHash, + title: sourceContract.title, + slug: sourceContract.slug, + description: sourceContract.description, + short_description: sourceContract.short_description, + logo_uri: params.sourceCatalogLogosUris.get(sourceContract.id) ?? undefined, + use_cases: [...sourceContract.use_cases], + verified: sourceContract.verified, + last_verified_date: sourceContract.last_verified_date ?? undefined, + playbook_supported: sourceContract.playbook_supported, + max_confidence_level: sourceContract.max_confidence_level, + support_version: sourceContract.support_version ?? undefined, + subscription_link: sourceContract.subscription_link ?? undefined, + source_code: sourceContract.source_code ?? undefined, + manager_supported: sourceContract.manager_supported, + version: sourceContract.container_version, + image: sourceContract.container_image, + connector_type: sourceContract.container_type, + config_schema: sourceContract.config_schema, + license_type: sourceContract.license_type ?? undefined, + solution_categories: sourceContract.solution_categories, + contact: sourceContract.contact ?? undefined, + }; + contractsCreations.push(contractCreation); + return; + } + if (sourceContractHash === currentContract.content_hash) { + return; + } + const contractUpdate: CatalogContractUpdate = { + catalog_id: params.sourceCatalog.id, + contract_id: sourceContract.id, + content_hash: sourceContractHash, + title: sourceContract.title, + slug: sourceContract.slug, + description: sourceContract.description, + short_description: sourceContract.short_description, + logo_uri: params.sourceCatalogLogosUris.get(sourceContract.id) ?? null, + use_cases: [...sourceContract.use_cases], + verified: sourceContract.verified, + last_verified_date: sourceContract.last_verified_date, + playbook_supported: sourceContract.playbook_supported, + max_confidence_level: sourceContract.max_confidence_level, + support_version: sourceContract.support_version, + subscription_link: sourceContract.subscription_link, + manager_supported: sourceContract.manager_supported, + version: sourceContract.container_version, + image: sourceContract.container_image, + connector_type: sourceContract.container_type, + config_schema: sourceContract.config_schema, + solution_categories: sourceContract.solution_categories, + source_code: sourceContract.source_code, + license_type: sourceContract.license_type, + contact: sourceContract.contact, + }; + contractsUpdates.push(contractUpdate); + }); + + params.currentContracts.forEach((currentContract) => { + if (!sourceContractsIds.has(currentContract.contract_id)) { + contractsDeletions.push({ idToDelete: currentContract.id }); + return; + } + }); + const catalogManifestUpsert: CatalogManifestUpsert = { + revision: params.revision, + source_uri: params.sourceConfig.uri, + catalog_id: params.sourceCatalog.id, + name: params.sourceCatalog.name, + description: params.sourceCatalog.description, + version: params.sourceCatalog.version, + }; + return { + contractsCreations, + contractsUpdates, + contractsDeletions, + catalogManifestUpsert, + }; +}; + +const synchronizeCatalog = async (context: AuthContext, user: AuthUser, sourceConfig: CatalogSyncSourceConfig) => { + try { + // Fetch source catalog + const sourceCatalog = await fetchSourceCatalog(sourceConfig); + // Find existing persisted data in the database corresponding to the catalog id + const currentCatalog = await findCatalogManifestByCatalogId(context, user, sourceCatalog.id); + const currentRevision = currentCatalog?.revision; + if (currentCatalog) { + logApp.debug('[OPENCTI-MODULE] Found existing persisted catalog', { + sourceKind: sourceConfig.kind, + catalogId: currentCatalog.catalog_id, + revision: currentRevision, + module: 'catalog', + }); + } else { + logApp.debug('[OPENCTI-MODULE] New catalog source', { + sourceKind: sourceConfig.kind, + module: 'catalog', + }); + } + if (currentCatalog && currentCatalog.id !== sourceCatalog.id) { + logApp.warn('[OPENCTI-MODULE] Same catalog source with different ID', { + sourceKind: sourceConfig.kind, + knownId: currentCatalog.id, + newId: sourceCatalog.id, + module: 'catalog', + }); + } + // Compute revision and compare with persisted, return early if equal. + const revision = computeCatalogRevision(sourceCatalog); + if (currentRevision && revision === currentRevision) { + logApp.info('[OPENCTI-MODULE] Catalog manager manifest unchanged (revision match)', { + sourceKind: sourceConfig.kind, + catalogId: sourceCatalog.id, + revision, + module: 'catalog', + }); + return { + synced: false, + catalogId: sourceCatalog.id, + } as const; + } + // Fetch all current contracts, logos & compute sync diff + const currentContracts = currentCatalog + ? await findCatalogContractsByCatalogId(context, user, currentCatalog.id) + : new Map(); + const currentLogos = await listCatalogContractLogos(); + const logoSyncOps = computeCatalogLogosSyncOps(sourceCatalog, currentLogos); + const sourceCatalogContractsHashes = sourceCatalog.contracts.reduce((hashesMap, contract) => { + hashesMap.set(contract.id, computeContractContentHash(contract)); + return hashesMap; + }, new Map()); + const catalogSyncDiff = computeCatalogSyncOps({ + revision, + sourceConfig, + sourceCatalog, + sourceCatalogContractsHashes, + sourceCatalogLogosUris: logoSyncOps.sourceCatalogLogosUris, + currentCatalog, + currentContracts, + }); + // Persist refreshed catalog, contracts & logos + await uploadCatalogContractLogos(logoSyncOps.uploadOperations); + await insertCatalogContracts(context, user, catalogSyncDiff.contractsCreations); + if (catalogSyncDiff.contractsUpdates.length > 0) { + // Usually there would be no updates of catalog contracts given a change + // would be treated in a new version. This could be the result of a previous + // failure being compensated or an issue in the release process. + // Something's fishy if this occurs too frequently. + logApp.warn('[OPENCTI-MODULE] Catalog contracts updates', { + catalogId: sourceCatalog.id, + count: catalogSyncDiff.contractsUpdates.length, + module: 'catalog', + }); + await updateCatalogContracts(context, user, catalogSyncDiff.contractsUpdates); + } + await deleteCatalogContracts(context, user, catalogSyncDiff.contractsDeletions); + await upsertCatalog(context, user, catalogSyncDiff.catalogManifestUpsert); + const isNotNil = (str: string | null | undefined): str is string => Boolean(str); + let usedLogos = catalogSyncDiff.contractsCreations.map(({ logo_uri }) => logo_uri).filter(isNotNil); + usedLogos = usedLogos.concat(catalogSyncDiff.contractsUpdates.map(({ logo_uri }) => logo_uri).filter(isNotNil)); + return { + synced: true as const, + catalogId: sourceCatalog.id, + usedLogos: new Set(usedLogos), + }; + } catch (exception) { + logApp.error('[OPENCTI-MODULE] [catalog] Error while syncing catalog', { + cause: exception, + }); + return { synced: false, error: true } as const; + } +}; + +const initSyncSources = () => { + // Build catalog sources configs from env vars/app settings + const sources: CatalogSyncSourceConfig[] = [{ + kind: 'embedded' as const, + uri: 'embedded', + }]; + const CUSTOM_CATALOGS: string[] = conf.get('app:custom_catalogs') ?? []; + if (CUSTOM_CATALOGS) { + sources.push(...CUSTOM_CATALOGS.map((customCatalog) => ({ + kind: 'local' as const, + filepath: customCatalog, + uri: 'file:///' + customCatalog, // TODO: normalize + }))); + } + return sources; +}; + +const cleanupObsoleteCatalogs = async (context: AuthContext, syncedCatalogs: string[]) => { + const obsoleteCatalogs = await findCatalogs(context, SYSTEM_USER, syncedCatalogs); + if (obsoleteCatalogs.length) { + // The app config changed resulting in a catalog not being synced anymore + logApp.warn('[OPENCTI-MODULE] Deleting obsolete catalogs', { + module: 'catalog', + deletedCatalogIds: obsoleteCatalogs.map((catalog) => catalog.catalog_id), + }); + await deleteCatalogs(context, obsoleteCatalogs); + } +}; + +export const synchronizeCatalogs = async (context: AuthContext, user: AuthUser) => { + const sources = initSyncSources(); + logApp.debug('[OPENCTI-MODULE] Synchronizing catalogs', { + count: sources.length, + module: 'catalog', + }); + const syncedCatalogs: string[] = []; + const syncedCatalogsWithChanges: string[] = []; + // Sync catalogs from sources + for (const source of sources) { + const result = await synchronizeCatalog(context, user, source); + if (!result.error) { + syncedCatalogs.push(result.catalogId); + if (result.synced) { + syncedCatalogsWithChanges.push(result.catalogId); + } + } + }; + // Cleanup obsolete catalogs only if no sync failed + if (syncedCatalogs.length === sources.length) { + await cleanupObsoleteCatalogs(context, syncedCatalogs); + } + return syncedCatalogsWithChanges; +}; diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-source-gateway.ts b/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-source-gateway.ts new file mode 100644 index 000000000000..e562e53c4d25 --- /dev/null +++ b/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-source-gateway.ts @@ -0,0 +1,100 @@ +import { readFile } from 'node:fs/promises'; +import type { CatalogContractDtoV0, CatalogDefinition, CatalogDtoV0 } from '../catalog-types'; +import type { CatalogContractSyncSource, CatalogSyncSource, CatalogSyncSourceConfig } from './catalog-sync-types'; +import { logApp } from '../../../config/conf'; +import { isEmptyField } from '../../../database/utils'; +import { UnsupportedError } from '../../../config/errors'; +import { getOrCompileValidator } from '../catalog-domain'; + +interface CatalogSyncSourceAdapter { + fetch: () => Promise; +} + +class EmbeddedCatalogSyncSource implements CatalogSyncSourceAdapter { + constructor(private _sourceConfig: Extract) {} + + async fetch() { + const data = await import('../../../__generated__/opencti-manifest.json'); + return data.default as unknown as CatalogDefinition; + } +} + +class LocalCatalogSyncSource implements CatalogSyncSourceAdapter { + constructor(private sourceConfig: Extract) {} + + async fetch() { + const catalog = await readFile(this.sourceConfig.filepath, { encoding: 'utf8', flag: 'r' }); + return JSON.parse(catalog); + } +} + +const getCatalogSourceAdapter = (config: CatalogSyncSourceConfig): CatalogSyncSourceAdapter => { + switch (config.kind) { + case 'local': { + return new LocalCatalogSyncSource(config); + } + case 'embedded': { + return new EmbeddedCatalogSyncSource(config); + } + default: + throw new Error(`Unknown catalog sync source kind (${JSON.stringify(config)})`); + }; +}; + +const validateSyncSource = (syncSource: CatalogSyncSource) => { + // Validate each contract + for (let contractIndex = 0; contractIndex < syncSource.contracts.length; contractIndex += 1) { + const contract = syncSource.contracts[contractIndex]; + if (contract.manager_supported) { + if (!contract.config_schema) { + logApp.warn('A contract has manager_supported=true but is missing config_schema', { contractTitle: contract.title }); + } else { + if (isEmptyField(contract.container_image)) { + throw UnsupportedError('Contract must define container_image field', { contractTitle: contract.title }); + } + if (isEmptyField(contract.container_type)) { + throw UnsupportedError('Contract must define container_type field', { contractTitle: contract.title }); + } + + if (contract.config_schema) { + const jsonValidation = { + type: contract.config_schema.type, + properties: contract.config_schema.properties, + required: contract.config_schema.required, + additionalProperties: contract.config_schema.additionalProperties, + }; + try { + getOrCompileValidator(`catalog-contract:${syncSource.id}:${contract.slug}`, jsonValidation); + } catch (err) { + throw UnsupportedError('Contract must be a valid json schema definition', { cause: err }); + } + } + } + } + } +}; + +export const mapCatalogContractDtoToCatalogContractSyncSource = ( + contractDto: CatalogContractDtoV0, +): CatalogContractSyncSource => { + return { + id: `${contractDto.slug}-${contractDto.container_version}`, + ...contractDto, + }; +}; + +export const mapCatalogDtoToCatalogSyncSource = (catalog: CatalogDtoV0) => { + const syncSource = { + ...catalog, + contracts: catalog.contracts.map(mapCatalogContractDtoToCatalogContractSyncSource), + }; + return syncSource; +}; + +export const fetchSourceCatalog = async (sourceConfig: CatalogSyncSourceConfig): Promise => { + const adapter = getCatalogSourceAdapter(sourceConfig); + const raw = await adapter.fetch(); + const syncSource = mapCatalogDtoToCatalogSyncSource(raw); + validateSyncSource(syncSource); + return syncSource; +}; diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-types.ts b/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-types.ts new file mode 100644 index 000000000000..21d244afad45 --- /dev/null +++ b/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-types.ts @@ -0,0 +1,22 @@ +import type { CatalogContractDtoV0 } from '../catalog-types'; + +export type CatalogSyncSourceConfig = { + kind: 'embedded'; + uri: 'embedded'; +} | { + kind: 'local'; + filepath: string; + uri: string; +}; + +export type CatalogContractSyncSource = CatalogContractDtoV0 & { + id: string; // slug-version +}; + +export interface CatalogSyncSource { + id: string; + name: string; + description: string; + version: string; + contracts: Array; +} diff --git a/opencti-platform/opencti-graphql/src/modules/connector/connector-domain.ts b/opencti-platform/opencti-graphql/src/modules/connector/connector-domain.ts new file mode 100644 index 000000000000..f5b0a4fa733d --- /dev/null +++ b/opencti-platform/opencti-graphql/src/modules/connector/connector-domain.ts @@ -0,0 +1,142 @@ +import semver from 'semver'; +import { logApp } from '../../config/conf'; +import { publishUserAction } from '../../listener/UserActionListener'; +import type { AuthContext, AuthUser } from '../../types/user'; +import { findLatestCompatibleCatalogContractBySlug } from '../catalog/catalog-repository'; +import { findManagedConnectorsByCatalogId } from './connector-repository'; +import type { BasicStoreEntityConnector } from '../../types/connector'; +import { patchAttribute } from '../../database/middleware'; +import { ENTITY_TYPE_CONNECTOR } from '../../schema/internalObject'; + +const autoUpgradeManagedConnector = async ( + context: AuthContext, + user: AuthUser, + managedConnector: BasicStoreEntityConnector, +) => { + const { manager_upgrade_strategy, manager_contract } = managedConnector; + // Currently we only support the "upgrade to latest compatible version" strategy + if (manager_upgrade_strategy !== 'latest') { + return; + } + if (!manager_contract) { + logApp.warn('[OPENCTI-MODULE] Inconsistent connector data, unable to find manager_contract on managed connector', { + module: 'connector', + connectorId: managedConnector.id, + }); + return; + } + const { slug, version, content_hash } = manager_contract; + try { + const latestCompatibleContract = await findLatestCompatibleCatalogContractBySlug(context, user, slug); + if (!latestCompatibleContract) { + // Warning: we're running a connector that's not compatible anymore but + // there's no replacement version compatible ! + // TODO: stop the connector and block the restart from the UI, or delete + // the connector ? + return; + } + if (semver.eq(version, latestCompatibleContract.version) + && content_hash === latestCompatibleContract.content_hash) { + logApp.debug('[OPENCTI-MODULE] Managed connector already uses latest compatible version', { + module: 'connector', + connectorId: managedConnector.id, + version, + }); + return; + } + // Update connector + const patch: Partial = { + manager_contract: latestCompatibleContract, + manager_contract_image: latestCompatibleContract.image, + }; + await patchAttribute(context, user, managedConnector.id, ENTITY_TYPE_CONNECTOR, patch); + if (semver.lt(version, latestCompatibleContract.version)) { + logApp.info('[OPENCTI-MODULE] Upgraded connector to latest compatible version', { + module: 'connector', + connectorId: managedConnector.id, + contractSlug: slug, + previousVersion: version, + newVersion: latestCompatibleContract.version, + }); + // Activity log + // Unsure how correct this is. Maybe the context_data is too big here. + void publishUserAction({ + event_type: 'mutation', + event_access: 'extended', + event_scope: 'update', + message: 'Upgraded connector to latest compatible version', + user, + context_data: { + entity_type: ENTITY_TYPE_CONNECTOR, + id: managedConnector.id, + input: patch, + }, + }); + } else if (semver.gt(version, latestCompatibleContract.version)) { + logApp.info('[OPENCTI-MODULE] Downgraded connector to latest compatible version', { + module: 'connector', + connectorId: managedConnector.id, + contractSlug: slug, + previousVersion: version, + newVersion: latestCompatibleContract.version, + }); + // Activity log + // Unsure how correct this is. Maybe the context_data is too big here. + void publishUserAction({ + event_type: 'mutation', + event_access: 'extended', + event_scope: 'update', + message: 'Downgraded connector to latest compatible version', + user, + context_data: { + entity_type: ENTITY_TYPE_CONNECTOR, + id: managedConnector.id, + input: patch, + }, + }); + } else if (semver.eq(version, latestCompatibleContract.version)) { + // Shouldn't happen: either a Release issue or a logic/code error. + logApp.warn('[OPENCTI-MODULE] Inconsistent connector data, same connector version with different contract content hash', { + module: 'connector', + contractSlug: slug, + contractVersion: version, + }); + // Activity log + // Unsure how correct this is. Maybe the context_data is too big here. + void publishUserAction({ + event_type: 'mutation', + event_access: 'extended', + event_scope: 'update', + message: 'Upgraded connector to latest compatible identical version', + user, + context_data: { + entity_type: ENTITY_TYPE_CONNECTOR, + id: managedConnector.id, + input: patch, + }, + }); + } else { + throw new Error('Unexpected case when comparing connector contract versions'); + } + } catch (exception) { + logApp.error('[OPENCTI-MODULE] Failed to auto-upgrade connector to latest compatible version', { + module: 'connector', + contractSlug: slug, + contractVersion: version, + cause: exception, + }); + } +}; + +export const autoUpgradeManagedConnectors = async ( + context: AuthContext, + user: AuthUser, + synchedCatalogs: string[], +) => { + for (const catalogId of synchedCatalogs) { + const managedConnectors = await findManagedConnectorsByCatalogId(context, user, catalogId); + for (const managedConnector of managedConnectors) { + await autoUpgradeManagedConnector(context, user, managedConnector); + }; + }; +}; diff --git a/opencti-platform/opencti-graphql/src/modules/connector/connector-repository.ts b/opencti-platform/opencti-graphql/src/modules/connector/connector-repository.ts new file mode 100644 index 000000000000..b88a6991b32a --- /dev/null +++ b/opencti-platform/opencti-graphql/src/modules/connector/connector-repository.ts @@ -0,0 +1,23 @@ +import { fullEntitiesList } from '../../database/middleware-loader'; +import { FilterMode, FilterOperator } from '../../generated/graphql'; +import { ENTITY_TYPE_CONNECTOR } from '../../schema/internalObject'; +import type { BasicStoreEntityConnector } from '../../types/connector'; +import type { AuthContext, AuthUser } from '../../types/user'; + +export const findManagedConnectorsByCatalogId = async ( + context: AuthContext, + user: AuthUser, + catalogId: string, +) => { + return await fullEntitiesList(context, user, [ENTITY_TYPE_CONNECTOR], { + filters: { + filters: [{ + key: ['catalog_id'], + values: [catalogId], + operator: FilterOperator.Eq, + }], + filterGroups: [], + mode: FilterMode.And, + }, + }); +}; diff --git a/opencti-platform/opencti-graphql/src/schema/schema-attributes.ts b/opencti-platform/opencti-graphql/src/schema/schema-attributes.ts index 2d907516deee..68bec6fa36a1 100644 --- a/opencti-platform/opencti-graphql/src/schema/schema-attributes.ts +++ b/opencti-platform/opencti-graphql/src/schema/schema-attributes.ts @@ -2,7 +2,7 @@ import * as R from 'ramda'; import { GraphQLDateTime } from 'graphql-scalars'; import { RULE_PREFIX } from './general'; import { FunctionalError, UnsupportedError } from '../config/errors'; -import type { AttributeDefinition, AttrType, ComplexAttributeWithMappings, MappingDefinition } from './attribute-definition'; +import type { AttributeDefinition, AttrType, BasicStoreAttribute, ComplexAttributeWithMappings, MappingDefinition, RawObjectAttribute } from './attribute-definition'; import { shortStringFormats } from './attribute-definition'; import { getParentTypes } from './schemaUtils'; import { isFeatureEnabled } from '../config/conf'; @@ -25,7 +25,7 @@ const isDateAttributeDefinition = (schemaDef: AttributeDefinition) => schemaDef. const isNonFlatObjectAttributeDefinition = (schemaDef: AttributeDefinition): schemaDef is ComplexAttributeWithMappings => { // handy typeguard return schemaDef.type === 'object' && schemaDef.format !== 'flat'; }; -const isRawObjectAttributeDefinition = (schemaDef: AttributeDefinition): schemaDef is ComplexAttributeWithMappings => { // handy typeguard +const isRawObjectAttributeDefinition = (schemaDef: AttributeDefinition): schemaDef is RawObjectAttribute => { // handy typeguard return schemaDef.type === 'object' && schemaDef.format === 'raw'; }; diff --git a/opencti-platform/opencti-graphql/src/telemetry/TelemetryMeterManager.ts b/opencti-platform/opencti-graphql/src/telemetry/TelemetryMeterManager.ts index 2a87246185be..ebb93c05fdc5 100644 --- a/opencti-platform/opencti-graphql/src/telemetry/TelemetryMeterManager.ts +++ b/opencti-platform/opencti-graphql/src/telemetry/TelemetryMeterManager.ts @@ -1,6 +1,7 @@ import { MeterProvider } from '@opentelemetry/sdk-metrics'; import type { ObservableResult } from '@opentelemetry/api'; import { ValueType } from '@opentelemetry/api'; +import type { CatalogContractEntityFields } from '../modules/catalog/catalog-types'; export const TELEMETRY_SERVICE_NAME = 'opencti-telemetry'; @@ -32,6 +33,7 @@ export const normalizeTelemetryTags = (rawTags: string | null | undefined): stri export interface ConnectorIdentitySource { catalog_id?: string | null; manager_contract_image?: string | null; + manager_contract?: CatalogContractEntityFields | null; name?: string | null; connector_type?: string | null; } @@ -75,13 +77,12 @@ export const stripImageToRepositoryPath = (imageReference: string | null | undef // available identity, flagged managed=false. export const computeActiveConnectorsByIdentity = ( activeConnectors: ConnectorIdentitySource[], - contractsByImage: ReadonlyMap, ): DimensionalGaugeItem[] => { const connectorsByIdentity = new Map(); activeConnectors.forEach((connector) => { const isManaged = (connector.catalog_id ?? '').length > 0; const slug = isManaged - ? (contractsByImage.get(connector.manager_contract_image ?? '')?.slug ?? stripImageToRepositoryPath(connector.manager_contract_image)) + ? (connector.manager_contract?.slug ?? stripImageToRepositoryPath(connector.manager_contract_image)) : (connector.name ?? '').trim().toLowerCase(); if (slug.length === 0) { return; diff --git a/opencti-platform/opencti-graphql/src/types/connector.d.ts b/opencti-platform/opencti-graphql/src/types/connector.d.ts index c5afb8ec7191..0e32a1fd26e0 100644 --- a/opencti-platform/opencti-graphql/src/types/connector.d.ts +++ b/opencti-platform/opencti-graphql/src/types/connector.d.ts @@ -1,3 +1,4 @@ +import type { ConnectorContractConfiguration } from '../generated/graphql'; import type { CatalogContractEntityFields } from '../modules/catalog/catalog-types'; import type { BasicStoreEntity, StoreEntity } from './store'; @@ -30,6 +31,7 @@ export interface BasicStoreEntityConnector extends StoreEntity { // region composer (set only on composer-managed connectors) catalog_id?: string; manager_contract_image?: string; + manager_contract_configuration?: ConnectorContractConfiguration[]; manager_contract?: ConnectorManagerContract; manager_upgrade_strategy?: string; // endregion diff --git a/opencti-platform/opencti-graphql/src/utils/data-url.ts b/opencti-platform/opencti-graphql/src/utils/data-url.ts index 9ba23b1a4643..863025780a23 100644 --- a/opencti-platform/opencti-graphql/src/utils/data-url.ts +++ b/opencti-platform/opencti-graphql/src/utils/data-url.ts @@ -29,4 +29,4 @@ export const parseDataUrl = (dataUrl: string) => { base64Encoded, data, }; -}; \ No newline at end of file +}; diff --git a/opencti-platform/opencti-graphql/tests/01-unit/domain/catalog-domain/ajvValidatorCache-test.ts b/opencti-platform/opencti-graphql/tests/01-unit/domain/catalog-domain/ajvValidatorCache-test.ts index 0b92925786ba..93f93e72296c 100644 --- a/opencti-platform/opencti-graphql/tests/01-unit/domain/catalog-domain/ajvValidatorCache-test.ts +++ b/opencti-platform/opencti-graphql/tests/01-unit/domain/catalog-domain/ajvValidatorCache-test.ts @@ -44,7 +44,6 @@ describe('catalog-domain - AJV validator compilation cache', () => { }); beforeEach(() => { - catalogDomain.resetCatalogs(); compileSpy.mockClear(); }); @@ -88,19 +87,6 @@ describe('catalog-domain - AJV validator compilation cache', () => { expect(compileSpy).toHaveBeenCalledTimes(2); }); - it('should invalidate the validator cache on resetCatalogs()', () => { - const contract = buildContract(); - const config = buildConfig({ IPINFO_TOKEN: 'token-value' }); - - catalogDomain.validateContractConfigurations(config, contract); - expect(compileSpy).toHaveBeenCalledTimes(1); - - catalogDomain.resetCatalogs(); - - catalogDomain.validateContractConfigurations(config, contract); - expect(compileSpy).toHaveBeenCalledTimes(2); - }); - it('should throw a formatted validation error without caching a broken state', () => { const contract = buildContract(); diff --git a/opencti-platform/opencti-graphql/tests/01-unit/domain/catalog-domain/connector-queue-threshold-test.ts b/opencti-platform/opencti-graphql/tests/01-unit/domain/catalog-domain/connector-queue-threshold-test.ts index 961d30a1267e..8748ca5dd4fc 100644 --- a/opencti-platform/opencti-graphql/tests/01-unit/domain/catalog-domain/connector-queue-threshold-test.ts +++ b/opencti-platform/opencti-graphql/tests/01-unit/domain/catalog-domain/connector-queue-threshold-test.ts @@ -29,23 +29,26 @@ const createMitreContract = (): CatalogContract => ({ MITRE_URL: { type: 'string', description: 'MITRE ATT&CK URL', - default: 'https://attack.mitre.org' + default: 'https://attack.mitre.org', }, CONNECTOR_QUEUE_THRESHOLD: { default: null, description: 'Connector queue max size in Mbytes. Default to 500.', exclusiveMinimum: 0, - type: 'integer' + type: 'integer', } as any, // Use any to bypass TypeScript constraint for testing INTERVAL: { type: 'integer', description: 'Interval in hours', - default: 24 - } + default: 24, + }, }, required: ['MITRE_URL'], // CONNECTOR_QUEUE_THRESHOLD is NOT required - additionalProperties: false - } + additionalProperties: false, + }, + license_type: null, + solution_categories: [], + contact: null, }); describe('CONNECTOR_QUEUE_THRESHOLD bug fix', () => { @@ -71,16 +74,16 @@ mwIDAQAB API_KEY: { type: 'string', format: 'password', - description: 'API key for authentication' - } as any // Use any to bypass TypeScript constraint for testing + description: 'API key for authentication', + } as any, // Use any to bypass TypeScript constraint for testing }, - required: ['MITRE_URL', 'API_KEY'] - } + required: ['MITRE_URL', 'API_KEY'], + }, }; const configurations: ContractConfigInput[] = [ { key: 'MITRE_URL', value: 'https://attack.mitre.org' }, - { key: 'API_KEY', value: 'my-secret-api-key' } + { key: 'API_KEY', value: 'my-secret-api-key' }, ]; const result = computeConnectorTargetContract(configurations, contractWithPassword, publicKey); @@ -107,16 +110,16 @@ mwIDAQAB PROXY_PASSWORD: { type: 'string', format: 'password', - description: 'Optional proxy password' - } as any // Use any to bypass TypeScript constraint for testing - } + description: 'Optional proxy password', + } as any, // Use any to bypass TypeScript constraint for testing + }, // PROXY_PASSWORD is not in required array - } + }, }; // Configuration without the optional password const configurations: ContractConfigInput[] = [ - { key: 'MITRE_URL', value: 'https://attack.mitre.org' } + { key: 'MITRE_URL', value: 'https://attack.mitre.org' }, // PROXY_PASSWORD not provided ]; @@ -135,33 +138,33 @@ mwIDAQAB properties: { USERNAME: { type: 'string', - description: 'Username' + description: 'Username', } as any, // Use any to bypass TypeScript constraint for testing PASSWORD: { type: 'string', format: 'password', - description: 'User password' + description: 'User password', } as any, // Use any to bypass TypeScript constraint for testing API_KEY: { type: 'string', format: 'password', - description: 'API key' + description: 'API key', } as any, // Use any to bypass TypeScript constraint for testing PROXY_PASSWORD: { type: 'string', format: 'password', - description: 'Proxy password' - } as any // Use any to bypass TypeScript constraint for testing + description: 'Proxy password', + } as any, // Use any to bypass TypeScript constraint for testing }, - required: ['USERNAME', 'PASSWORD', 'API_KEY'] - } + required: ['USERNAME', 'PASSWORD', 'API_KEY'], + }, }; const configurations: ContractConfigInput[] = [ { key: 'USERNAME', value: 'admin' }, { key: 'PASSWORD', value: 'user-password' }, { key: 'API_KEY', value: 'api-key-value' }, - { key: 'PROXY_PASSWORD', value: 'proxy-pass' } + { key: 'PROXY_PASSWORD', value: 'proxy-pass' }, ]; const result = computeConnectorTargetContract(configurations, contractWithMultiplePasswords, publicKey); @@ -195,11 +198,11 @@ mwIDAQAB SECRET: { type: 'string', format: 'password', - description: 'Secret key' - } as any // Use any to bypass TypeScript constraint for testing + description: 'Secret key', + } as any, // Use any to bypass TypeScript constraint for testing }, - required: ['MITRE_URL', 'SECRET'] - } + required: ['MITRE_URL', 'SECRET'], + }, }; // Configuration with encrypted password @@ -208,8 +211,8 @@ mwIDAQAB { key: 'SECRET', value: 'AQEAAf8AAABAwOL5+encrypted_value', // Already encrypted - encrypted: true - } + encrypted: true, + }, ]; // Should validate without throwing @@ -228,16 +231,16 @@ mwIDAQAB PASSWORD: { type: 'string', format: 'password', - description: 'Optional password' - } as any // Use any to bypass TypeScript constraint for testing - } + description: 'Optional password', + } as any, // Use any to bypass TypeScript constraint for testing + }, // PASSWORD is NOT in required array, making it optional - } + }, }; const configurations: ContractConfigInput[] = [ { key: 'MITRE_URL', value: 'https://attack.mitre.org' }, - { key: 'PASSWORD', value: '' } // Empty password + { key: 'PASSWORD', value: '' }, // Empty password ]; // Empty password should be handled (returns null) @@ -254,7 +257,7 @@ mwIDAQAB // Only providing required field const configurations: ContractConfigInput[] = [ - { key: 'MITRE_URL', value: 'https://attack.mitre.org' } + { key: 'MITRE_URL', value: 'https://attack.mitre.org' }, // CONNECTOR_QUEUE_THRESHOLD is not provided ]; @@ -274,7 +277,7 @@ mwIDAQAB // Configuration without the optional CONNECTOR_QUEUE_THRESHOLD const configurations: ConnectorContractConfiguration[] = [ - { key: 'MITRE_URL', value: 'https://attack.mitre.org' } + { key: 'MITRE_URL', value: 'https://attack.mitre.org' }, ]; // Should not throw validation error @@ -288,7 +291,7 @@ mwIDAQAB const configurations: ContractConfigInput[] = [ { key: 'MITRE_URL', value: 'https://attack.mitre.org' }, - { key: 'CONNECTOR_QUEUE_THRESHOLD', value: '1000' } + { key: 'CONNECTOR_QUEUE_THRESHOLD', value: '1000' }, ]; const result = computeConnectorTargetContract(configurations, contract, publicKey); @@ -296,7 +299,7 @@ mwIDAQAB expect(result).toHaveLength(3); // MITRE_URL, CONNECTOR_QUEUE_THRESHOLD, and INTERVAL (default) expect(result.find((c) => c.key === 'CONNECTOR_QUEUE_THRESHOLD')).toEqual({ key: 'CONNECTOR_QUEUE_THRESHOLD', - value: '1000' + value: '1000', }); }); @@ -305,7 +308,7 @@ mwIDAQAB const configurations: ContractConfigInput[] = [ { key: 'MITRE_URL', value: 'https://attack.mitre.org' }, - { key: 'CONNECTOR_QUEUE_THRESHOLD', value: 'not_a_number' } + { key: 'CONNECTOR_QUEUE_THRESHOLD', value: 'not_a_number' }, ]; expect(() => { @@ -318,7 +321,7 @@ mwIDAQAB // Test with INTERVAL which has a default value of 24 const configurations: ContractConfigInput[] = [ - { key: 'MITRE_URL', value: 'https://attack.mitre.org' } + { key: 'MITRE_URL', value: 'https://attack.mitre.org' }, // INTERVAL not provided, should use default ]; @@ -328,7 +331,7 @@ mwIDAQAB const intervalConfig = result.find((c) => c.key === 'INTERVAL'); expect(intervalConfig).toEqual({ key: 'INTERVAL', - value: '24' // default value converted to string + value: '24', // default value converted to string }); }); @@ -336,7 +339,7 @@ mwIDAQAB const contract = createMitreContract(); const configurations: ContractConfigInput[] = [ - { key: 'MITRE_URL', value: 'https://attack.mitre.org' } + { key: 'MITRE_URL', value: 'https://attack.mitre.org' }, ]; const result = computeConnectorTargetContract(configurations, contract, publicKey); @@ -355,25 +358,25 @@ mwIDAQAB REQUIRED_FIELD: { type: 'string', description: 'Required field', - default: 'default_value' + default: 'default_value', }, OPTIONAL_WITH_DEFAULT: { type: 'integer', description: 'Optional with default', - default: 100 + default: 100, }, OPTIONAL_NO_DEFAULT: { type: 'integer', - description: 'Optional without default' + description: 'Optional without default', // No default provided intentionally - } as any // Use any to bypass TypeScript constraint for testing + } as any, // Use any to bypass TypeScript constraint for testing }, - required: ['REQUIRED_FIELD'] - } + required: ['REQUIRED_FIELD'], + }, }; const configurations: ContractConfigInput[] = [ - { key: 'REQUIRED_FIELD', value: 'test' } + { key: 'REQUIRED_FIELD', value: 'test' }, // All optional fields omitted ]; @@ -385,7 +388,7 @@ mwIDAQAB // Should include optional field with non-null default expect(result.find((c) => c.key === 'OPTIONAL_WITH_DEFAULT')).toEqual({ key: 'OPTIONAL_WITH_DEFAULT', - value: '100' + value: '100', }); // Should NOT include optional fields without default diff --git a/opencti-platform/opencti-graphql/tests/01-unit/domain/catalog-domain/getCatalogsRaceCondition-test.ts b/opencti-platform/opencti-graphql/tests/01-unit/domain/catalog-domain/getCatalogsRaceCondition-test.ts deleted file mode 100644 index 2fb64f285e00..000000000000 --- a/opencti-platform/opencti-graphql/tests/01-unit/domain/catalog-domain/getCatalogsRaceCondition-test.ts +++ /dev/null @@ -1,151 +0,0 @@ -import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import fs from 'node:fs'; -import os from 'node:os'; -import path from 'node:path'; -import type { AuthContext, AuthUser } from '../../../../src/types/user'; - -const FAKE_CONTEXT = {} as unknown as AuthContext; -const FAKE_USER = {} as unknown as AuthUser; - -const validCatalogContent = (id: string) => JSON.stringify({ - id, - name: 'Race Condition Test Catalog', - description: 'Custom catalog used only for cache-stampede unit tests', - contracts: [ - { - title: 'Race Condition Test Connector', - slug: 'race-condition-test-connector', - manager_supported: false, - }, - ], -}); - -let tmpDir: string; -let catalogFilePath: string; - -const loadFreshCatalogDomain = async (customCatalogPaths: string[]) => { - process.env.APP__CUSTOM_CATALOGS = JSON.stringify(customCatalogPaths); - vi.resetModules(); - return import('../../../../src/modules/catalog/catalog-domain'); -}; - -describe('catalog-domain - getCatalogs() / getSupportedContractsByImage() concurrency (cache stampede)', () => { - beforeEach(() => { - tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'catalog-race-test-')); - catalogFilePath = path.join(tmpDir, 'custom-catalog.json'); - }); - - afterEach(() => { - delete process.env.APP__CUSTOM_CATALOGS; - fs.rmSync(tmpDir, { recursive: true, force: true }); - }); - - afterAll(() => { - vi.resetModules(); - }); - - it('should return the exact same in-memory result across 20 concurrent calls mixing findCatalog() and getSupportedContractsByImage() (no duplicate build)', async () => { - fs.writeFileSync(catalogFilePath, validCatalogContent('race-test-catalog-1'), 'utf8'); - const catalogDomain = await loadFreshCatalogDomain([catalogFilePath]); - - const calls = Array.from({ length: 20 }, (_, index) => (index % 2 === 0 - ? catalogDomain.findCatalog(FAKE_CONTEXT, FAKE_USER) - : catalogDomain.getSupportedContractsByImage())); - - const results = await Promise.all(calls); - - // getSupportedContractsByImage() is memoized as a single Promise, - // so the exact same Map instance must be returned to every caller. - const contractMaps = results.filter((_, index) => index % 2 === 1) as Map[]; - contractMaps.forEach((map) => { - expect(map).toBe(contractMaps[0]); - }); - - const catalogArrays = results.filter((_, index) => index % 2 === 0) as { id: string }[][]; - const firstCatalogRef = catalogArrays[0].find((c) => c.id === 'race-test-catalog-1'); - catalogArrays.forEach((catalogs) => { - const catalogRef = catalogs.find((c) => c.id === 'race-test-catalog-1'); - expect(catalogRef).toBe(firstCatalogRef); - }); - }); - - it('should reuse the same in-flight build when getCatalogs() is triggered indirectly from two different entry points at once', async () => { - fs.writeFileSync(catalogFilePath, validCatalogContent('race-test-catalog-2'), 'utf8'); - const catalogDomain = await loadFreshCatalogDomain([catalogFilePath]); - - // Cold start: both entry points race to trigger the very first build. - const [catalogs, contractsByImage] = await Promise.all([ - catalogDomain.findCatalog(FAKE_CONTEXT, FAKE_USER), - catalogDomain.getSupportedContractsByImage(), - ]); - - expect(catalogs.find((c) => c.id === 'race-test-catalog-2')).toBeDefined(); - expect(contractsByImage).toBeInstanceOf(Map); - - // A subsequent call from either entry point must still resolve to the - // exact same underlying catalog object, proving there was only one build. - const catalogsAgain = await catalogDomain.findCatalog(FAKE_CONTEXT, FAKE_USER); - expect(catalogsAgain.find((c) => c.id === 'race-test-catalog-2')) - .toBe(catalogs.find((c) => c.id === 'race-test-catalog-2')); - }); - - it('should not read the catalog file again on a second sequential call (module-level cache)', async () => { - fs.writeFileSync(catalogFilePath, validCatalogContent('race-test-catalog-3'), 'utf8'); - const catalogDomain = await loadFreshCatalogDomain([catalogFilePath]); - - const first = await catalogDomain.findCatalog(FAKE_CONTEXT, FAKE_USER); - const second = await catalogDomain.findCatalog(FAKE_CONTEXT, FAKE_USER); - - expect(second.find((c) => c.id === 'race-test-catalog-3')) - .toBe(first.find((c) => c.id === 'race-test-catalog-3')); - }); - - it('should rebuild (and reflect real file changes) after resetCatalogs()', async () => { - fs.writeFileSync(catalogFilePath, validCatalogContent('race-test-catalog-4'), 'utf8'); - const catalogDomain = await loadFreshCatalogDomain([catalogFilePath]); - - const before = await catalogDomain.findCatalog(FAKE_CONTEXT, FAKE_USER); - const beforeRef = before.find((c) => c.id === 'race-test-catalog-4'); - expect(beforeRef).toBeDefined(); - - catalogDomain.resetCatalogs(); - - const after = await catalogDomain.findCatalog(FAKE_CONTEXT, FAKE_USER); - const afterRef = after.find((c) => c.id === 'race-test-catalog-4'); - - // Same logical catalog, but a genuinely new object graph -> proves a real rebuild happened. - expect(afterRef).toBeDefined(); - expect(afterRef).not.toBe(beforeRef); - expect(afterRef).toEqual(beforeRef); - }); - - it('should allow a retry after a failed build instead of caching the rejection forever (real file deleted then recreated)', async () => { - fs.writeFileSync(catalogFilePath, validCatalogContent('race-test-catalog-5'), 'utf8'); - const catalogDomain = await loadFreshCatalogDomain([catalogFilePath]); - - fs.unlinkSync(catalogFilePath); - await expect(catalogDomain.findCatalog(FAKE_CONTEXT, FAKE_USER)).rejects.toThrow(); - - fs.writeFileSync(catalogFilePath, validCatalogContent('race-test-catalog-5'), 'utf8'); - - const catalogs = await catalogDomain.findCatalog(FAKE_CONTEXT, FAKE_USER); - expect(catalogs.find((c) => c.id === 'race-test-catalog-5')).toBeDefined(); - }); - - it('should not duplicate work when 20 concurrent calls race right after a resetCatalogs()', async () => { - fs.writeFileSync(catalogFilePath, validCatalogContent('race-test-catalog-6'), 'utf8'); - const catalogDomain = await loadFreshCatalogDomain([catalogFilePath]); - - await catalogDomain.findCatalog(FAKE_CONTEXT, FAKE_USER); - catalogDomain.resetCatalogs(); - - const results = await Promise.all( - Array.from({ length: 20 }, () => catalogDomain.findCatalog(FAKE_CONTEXT, FAKE_USER)), - ); - - const firstRef = results[0].find((c) => c.id === 'race-test-catalog-6'); - results.forEach((catalogs) => { - expect(catalogs.find((c) => c.id === 'race-test-catalog-6')).toBe(firstRef); - }); - }); -}); diff --git a/opencti-platform/opencti-graphql/tests/01-unit/domain/catalog-domain/validateContractConfigurations-test.ts b/opencti-platform/opencti-graphql/tests/01-unit/domain/catalog-domain/validateContractConfigurations-test.ts index 728a3c47765d..1ee800b02e18 100644 --- a/opencti-platform/opencti-graphql/tests/01-unit/domain/catalog-domain/validateContractConfigurations-test.ts +++ b/opencti-platform/opencti-graphql/tests/01-unit/domain/catalog-domain/validateContractConfigurations-test.ts @@ -27,8 +27,11 @@ const createTestContract = (properties: any, required: string[] = []): CatalogCo type: 'object', properties, required, - additionalProperties: false - } + additionalProperties: false, + }, + license_type: null, + solution_categories: [], + contact: null, }); describe('validateContractConfigurations', () => { @@ -37,15 +40,15 @@ describe('validateContractConfigurations', () => { { name: { type: 'string' }, port: { type: 'integer' }, - enabled: { type: 'boolean' } + enabled: { type: 'boolean' }, }, - ['name', 'port', 'enabled'] + ['name', 'port', 'enabled'], ); const configurations: ConnectorContractConfiguration[] = [ { key: 'name', value: 'test-service' }, { key: 'port', value: '8080' }, - { key: 'enabled', value: 'true' } + { key: 'enabled', value: 'true' }, ]; // Should not throw @@ -58,13 +61,13 @@ describe('validateContractConfigurations', () => { const contract = createTestContract( { name: { type: 'string' }, - port: { type: 'integer' } + port: { type: 'integer' }, }, - ['name', 'port'] + ['name', 'port'], ); const configurations: ConnectorContractConfiguration[] = [ - { key: 'name', value: 'test-service' } + { key: 'name', value: 'test-service' }, // port is missing ]; @@ -84,13 +87,13 @@ describe('validateContractConfigurations', () => { const contract = createTestContract( { name: { type: 'string' }, - description: { type: 'string' } + description: { type: 'string' }, }, - ['name'] // only name is required + ['name'], // only name is required ); const configurations: ConnectorContractConfiguration[] = [ - { key: 'name', value: 'test-service' } + { key: 'name', value: 'test-service' }, // description is optional and not provided ]; @@ -103,14 +106,14 @@ describe('validateContractConfigurations', () => { const contract = createTestContract( { port: { type: 'integer' }, - enabled: { type: 'boolean' } + enabled: { type: 'boolean' }, }, - ['port', 'enabled'] + ['port', 'enabled'], ); const configurations: ConnectorContractConfiguration[] = [ { key: 'port', value: '8080' }, // string that can be coerced to integer - { key: 'enabled', value: 'true' } // string that can be coerced to boolean + { key: 'enabled', value: 'true' }, // string that can be coerced to boolean ]; expect(() => { @@ -121,13 +124,13 @@ describe('validateContractConfigurations', () => { it('should throw error for invalid type that cannot be coerced', () => { const contract = createTestContract( { - port: { type: 'integer' } + port: { type: 'integer' }, }, - ['port'] + ['port'], ); const configurations: ConnectorContractConfiguration[] = [ - { key: 'port', value: 'not_a_number' } + { key: 'port', value: 'not_a_number' }, ]; try { @@ -144,13 +147,13 @@ describe('validateContractConfigurations', () => { it('should handle array type in configuration', () => { const contract = createTestContract( { - tags: { type: 'array', items: { type: 'string' } } + tags: { type: 'array', items: { type: 'string' } }, }, - ['tags'] + ['tags'], ); const configurations: ConnectorContractConfiguration[] = [ - { key: 'tags', value: 'tag1,tag2,tag3' } + { key: 'tags', value: 'tag1,tag2,tag3' }, ]; // Arrays passed as comma-separated strings should now work @@ -163,13 +166,13 @@ describe('validateContractConfigurations', () => { it('should handle empty array from comma-separated string', () => { const contract = createTestContract( { - tags: { type: 'array', items: { type: 'string' } } + tags: { type: 'array', items: { type: 'string' } }, }, - [] // tags is optional + [], // tags is optional ); const configurations: ConnectorContractConfiguration[] = [ - { key: 'tags', value: '' } // empty string should become empty array + { key: 'tags', value: '' }, // empty string should become empty array ]; expect(() => { @@ -180,13 +183,13 @@ describe('validateContractConfigurations', () => { it('should handle array with spaces in comma-separated values', () => { const contract = createTestContract( { - scopes: { type: 'array', items: { type: 'string' } } + scopes: { type: 'array', items: { type: 'string' } }, }, - ['scopes'] + ['scopes'], ); const configurations: ConnectorContractConfiguration[] = [ - { key: 'scopes', value: 'greynoisefeed, toto , test ' } // with spaces + { key: 'scopes', value: 'greynoisefeed, toto , test ' }, // with spaces ]; // Should trim spaces and validate correctly @@ -200,15 +203,15 @@ describe('validateContractConfigurations', () => { { name: { type: 'string' }, optional1: { type: 'string' }, - optional2: { type: 'string' } + optional2: { type: 'string' }, }, - ['name'] + ['name'], ); const configurations: ConnectorContractConfiguration[] = [ { key: 'name', value: 'test' }, { key: 'optional1', value: undefined as any }, - { key: 'optional2', value: null as any } + { key: 'optional2', value: null as any }, ]; expect(() => { @@ -222,16 +225,16 @@ describe('validateContractConfigurations', () => { name: { type: 'string' }, port: { type: 'integer' }, enabled: { type: 'boolean' }, - timeout: { type: 'integer' } + timeout: { type: 'integer' }, }, - ['name', 'port', 'enabled'] + ['name', 'port', 'enabled'], ); const configurations: ConnectorContractConfiguration[] = [ { key: 'name', value: 'complex-service' }, { key: 'port', value: '3000' }, { key: 'enabled', value: 'false' }, - { key: 'timeout', value: '30' } + { key: 'timeout', value: '30' }, ]; expect(() => { @@ -242,9 +245,9 @@ describe('validateContractConfigurations', () => { it('should handle empty configuration array for schema with no required fields', () => { const contract = createTestContract( { - optional: { type: 'string' } + optional: { type: 'string' }, }, - [] // no required fields + [], // no required fields ); const configurations: ConnectorContractConfiguration[] = []; @@ -259,9 +262,9 @@ describe('validateContractConfigurations', () => { const contract = createTestContract( { username: { type: 'string' }, - password: { type: 'string', format: 'password' } + password: { type: 'string', format: 'password' }, }, - ['username', 'password'] + ['username', 'password'], ); const configurations: ConnectorContractConfiguration[] = [ @@ -269,8 +272,8 @@ describe('validateContractConfigurations', () => { { key: 'password', value: 'AQEAAf8AAABAwOL5+encrypted_base64_value', // Encrypted value - encrypted: true - } + encrypted: true, + }, ]; // Should validate encrypted passwords successfully @@ -282,16 +285,16 @@ describe('validateContractConfigurations', () => { it('should validate unencrypted password fields', () => { const contract = createTestContract( { - api_key: { type: 'string', format: 'password' } + api_key: { type: 'string', format: 'password' }, }, - ['api_key'] + ['api_key'], ); const configurations: ConnectorContractConfiguration[] = [ { key: 'api_key', - value: 'plain_text_api_key' // Unencrypted password (not tagged as encrypted) - } + value: 'plain_text_api_key', // Unencrypted password (not tagged as encrypted) + }, ]; // Should still validate unencrypted passwords @@ -305,27 +308,27 @@ describe('validateContractConfigurations', () => { { primary_password: { type: 'string', format: 'password' }, secondary_password: { type: 'string', format: 'password' }, - api_token: { type: 'string', format: 'password' } + api_token: { type: 'string', format: 'password' }, }, - ['primary_password', 'secondary_password'] + ['primary_password', 'secondary_password'], ); const configurations: ConnectorContractConfiguration[] = [ { key: 'primary_password', value: 'encrypted_primary', - encrypted: true + encrypted: true, }, { key: 'secondary_password', value: 'encrypted_secondary', - encrypted: true + encrypted: true, }, { key: 'api_token', value: 'optional_encrypted_token', - encrypted: true - } + encrypted: true, + }, ]; expect(() => { @@ -339,9 +342,9 @@ describe('validateContractConfigurations', () => { host: { type: 'string' }, port: { type: 'integer' }, password: { type: 'string', format: 'password' }, - ssl_enabled: { type: 'boolean' } + ssl_enabled: { type: 'boolean' }, }, - ['host', 'port', 'password'] + ['host', 'port', 'password'], ); const configurations: ConnectorContractConfiguration[] = [ @@ -350,9 +353,9 @@ describe('validateContractConfigurations', () => { { key: 'password', value: 'encrypted_password_value', - encrypted: true + encrypted: true, }, - { key: 'ssl_enabled', value: 'true' } + { key: 'ssl_enabled', value: 'true' }, ]; expect(() => { @@ -364,18 +367,18 @@ describe('validateContractConfigurations', () => { const contract = createTestContract( { username: { type: 'string' }, - count: { type: 'integer' } + count: { type: 'integer' }, }, - ['username', 'count'] + ['username', 'count'], ); const configurations: ConnectorContractConfiguration[] = [ { key: 'username', value: 'john_doe', - encrypted: true // Incorrectly marked as encrypted + encrypted: true, // Incorrectly marked as encrypted }, - { key: 'count', value: '42' } + { key: 'count', value: '42' }, ]; // Should validate even with incorrect encrypted flag @@ -387,9 +390,9 @@ describe('validateContractConfigurations', () => { it('should validate empty password field', () => { const contract = createTestContract( { - optional_password: { type: 'string', format: 'password' } + optional_password: { type: 'string', format: 'password' }, }, - [] // Password is optional + [], // Password is optional ); const configurations: ConnectorContractConfiguration[] = [ @@ -404,9 +407,9 @@ describe('validateContractConfigurations', () => { it('should fail validation for missing required password', () => { const contract = createTestContract( { - auth_token: { type: 'string', format: 'password' } + auth_token: { type: 'string', format: 'password' }, }, - ['auth_token'] // Password is required + ['auth_token'], // Password is required ); const configurations: ConnectorContractConfiguration[] = [ diff --git a/opencti-platform/opencti-graphql/tests/01-unit/manager/telemetryTags-test.ts b/opencti-platform/opencti-graphql/tests/01-unit/manager/telemetryTags-test.ts index ece545ad5632..0e29dd73a3c0 100644 --- a/opencti-platform/opencti-graphql/tests/01-unit/manager/telemetryTags-test.ts +++ b/opencti-platform/opencti-graphql/tests/01-unit/manager/telemetryTags-test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from 'vitest'; import { computeActiveConnectorsByIdentity, normalizeTelemetryTags, stripImageToRepositoryPath, type ConnectorIdentitySource } from '../../../src/telemetry/TelemetryMeterManager'; +import type { CatalogContractEntityFields } from '../../../src/modules/catalog/catalog-types'; describe('Telemetry tags normalization', () => { it('should return an empty string when no tags are configured', () => { @@ -67,6 +68,9 @@ describe('Active connectors by catalog identity', () => { const managed = (image: string, type = 'EXTERNAL_IMPORT'): ConnectorIdentitySource => ({ catalog_id: 'catalog-1', manager_contract_image: image, + manager_contract: { + slug: CONTRACTS.get(image)!.slug, + } as CatalogContractEntityFields, name: 'User Renamed Me', connector_type: type, }); @@ -77,11 +81,11 @@ describe('Active connectors by catalog identity', () => { }); it('should return no datapoint for an empty connector list', () => { - expect(computeActiveConnectorsByIdentity([], CONTRACTS)).toEqual([]); + expect(computeActiveConnectorsByIdentity([])).toEqual([]); }); it('should resolve managed connectors to the catalog contract slug, ignoring the user-set name', () => { - const items = computeActiveConnectorsByIdentity([managed('opencti/connector-mitre')], CONTRACTS); + const items = computeActiveConnectorsByIdentity([managed('opencti/connector-mitre')]); expect(items).toEqual([ { value: 1, attributes: { slug: 'mitre-att-ck', managed: 'true', type: 'EXTERNAL_IMPORT' } }, ]); @@ -90,7 +94,6 @@ describe('Active connectors by catalog identity', () => { it('should export the registry-stripped repository path for managed connectors whose image is not in the catalog', () => { const items = computeActiveConnectorsByIdentity( [managed('registry.private.corp:5000/team/custom-connector:1.2.3'), managed('opencti/connector-misp')], - CONTRACTS, ); expect(items).toHaveLength(2); // The datapoint stays visible, but the private registry hostname is stripped. @@ -104,25 +107,24 @@ describe('Active connectors by catalog identity', () => { }); it('should still skip managed connectors without any usable image reference', () => { - expect(computeActiveConnectorsByIdentity([managed(''), managed(' ')], CONTRACTS)).toEqual([]); + expect(computeActiveConnectorsByIdentity([managed(''), managed(' ')])).toEqual([]); }); it('should fall back to the trimmed/lowercased registered name for manual connectors', () => { - const items = computeActiveConnectorsByIdentity([manual(' My Custom Feed ')], CONTRACTS); + const items = computeActiveConnectorsByIdentity([manual(' My Custom Feed ')]); expect(items).toEqual([ { value: 1, attributes: { slug: 'my custom feed', managed: 'false', type: 'EXTERNAL_IMPORT' } }, ]); }); it('should skip manual connectors without a usable name', () => { - expect(computeActiveConnectorsByIdentity([manual(''), manual(' '), manual(null), manual(undefined)], CONTRACTS)).toEqual([]); + expect(computeActiveConnectorsByIdentity([manual(''), manual(' '), manual(null), manual(undefined)])).toEqual([]); }); it('should not collide identities when a freeform manual name contains separator-like characters', () => { // Under a naive 'slug|managed|type' string key these two would merge. const items = computeActiveConnectorsByIdentity( [manual('a|false', 't'), manual('a', 'false|t')], - CONTRACTS, ); expect(items).toHaveLength(2); expect(items).toContainEqual({ value: 1, attributes: { slug: 'a|false', managed: 'false', type: 't' } }); @@ -137,7 +139,6 @@ describe('Active connectors by catalog identity', () => { managed('opencti/connector-mitre', 'INTERNAL_ENRICHMENT'), manual('mitre-att-ck'), ], - CONTRACTS, ); expect(items).toContainEqual({ value: 2, attributes: { slug: 'mitre-att-ck', managed: 'true', type: 'EXTERNAL_IMPORT' } }); expect(items).toContainEqual({ value: 1, attributes: { slug: 'mitre-att-ck', managed: 'true', type: 'INTERNAL_ENRICHMENT' } }); diff --git a/opencti-platform/opencti-graphql/tests/03-integration/02-resolvers/connector-composer-test.ts b/opencti-platform/opencti-graphql/tests/03-integration/02-resolvers/connector-composer-test.ts index f5e4f95b3b8e..245cf03dcaad 100644 --- a/opencti-platform/opencti-graphql/tests/03-integration/02-resolvers/connector-composer-test.ts +++ b/opencti-platform/opencti-graphql/tests/03-integration/02-resolvers/connector-composer-test.ts @@ -8,8 +8,6 @@ import { wait } from '../../../src/database/utils'; import { XTMComposerMock } from '../../utils/XTMComposerMock'; import type { ApiConnector } from '../../utils/XTMComposerMock'; import { catalogHelper } from '../../utils/catalogHelper'; -import { resetCatalogs } from '../../../src/modules/catalog/catalog-domain'; -import * as UserActionListener from '../../../src/listener/UserActionListener'; const TEST_COMPOSER_ID = uuidv4(); const TEST_USER_CONNECTOR_ID: string = USER_CONNECTOR.id; // Initialize with default value @@ -259,9 +257,6 @@ describe('Connector Composer and Managed Connectors', () => { const testCatalogUrl = new URL('../../utils/opencti-manifest.json', import.meta.url); process.env.APP__CUSTOM_CATALOGS = JSON.stringify([fileURLToPath(testCatalogUrl)]); - // Reset catalogs to ensure test catalog is loaded - resetCatalogs(); - // Validate that we're using the test catalog catalogHelper.validateTestCatalog(); diff --git a/opencti-platform/opencti-graphql/tests/03-integration/02-resolvers/connector-migration-test.ts b/opencti-platform/opencti-graphql/tests/03-integration/02-resolvers/connector-migration-test.ts index 771f3747990b..82fd584d4c96 100644 --- a/opencti-platform/opencti-graphql/tests/03-integration/02-resolvers/connector-migration-test.ts +++ b/opencti-platform/opencti-graphql/tests/03-integration/02-resolvers/connector-migration-test.ts @@ -3,7 +3,7 @@ import gql from 'graphql-tag'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { registerConnector } from '../../../src/domain/connector'; import { ConnectorType } from '../../../src/generated/graphql'; -import * as catalogDomain from '../../../src/modules/catalog/catalog-domain'; +import * as catalogRepository from '../../../src/modules/catalog/catalog-repository'; import { ADMIN_USER, testContext } from '../../utils/testQuery'; import { queryAsAdmin } from '../../utils/testQueryHelper'; import { queryAsAdminWithSuccess } from '../../utils/testQueryHelper'; @@ -188,22 +188,11 @@ describe('Check connector migration', () => { }, }); - const contractFound = await catalogDomain.findContractByContainerImage(testContext, ADMIN_USER, 'opencti/connector-cve'); - if (!contractFound?.contract) { + const contractFound = await catalogRepository.findLatestCompatibleCatalogContractByImageName(testContext, ADMIN_USER, 'opencti/connector-cve'); + if (!contractFound) { throw new Error('Connector nist-nvd-cve container-image not found in catalog'); } - let contractParsed; - try { - contractParsed = JSON.parse(contractFound?.contract); - } catch { - throw new Error('Cannot parse nist-nvd-cve catalog'); - } - - if (!contractParsed) { - throw new Error('Contract nist-nvd-cve catalog is undefined'); - } - // same values excluded from catalog-domain const RUNTIME_KEYS = ['OPENCTI_TOKEN', 'CONNECTOR_ID', 'CONNECTOR_TYPE', 'HTTP_PROXY', 'HTTPS_PROXY', 'NO_PROXY', 'HTTPS_PROXY_REJECT_UNAUTHORIZED']; const managedConnector = managedConnectorResult.data.connectorMigrateToManaged; @@ -211,7 +200,7 @@ describe('Check connector migration', () => { rawConfig.filter((c: any) => !RUNTIME_KEYS.includes(c.key)); const actualConfig = rawConfig.filter((c: { key: string }) => !RUNTIME_KEYS.includes(c.key)); - const schemaProperties = contractParsed.config_schema.properties; + const schemaProperties = contractFound.config_schema.properties; const expectedKeys = Object.keys(schemaProperties); const actualKeys = actualConfig.map((c: { key: string }) => c.key); From e13b177d07091034a791cf873e4554d05799fbd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Tue, 18 Aug 2026 20:58:07 +0200 Subject: [PATCH 04/45] feat(catalog): add remote manifest sync with v1 normalization --- .../opencti-graphql/src/config/conf.js | 1 + .../opencti-graphql/src/manager/index.ts | 2 + .../src/manager/managerModule.ts | 1 + .../src/modules/catalog/catalog-http.ts | 11 +- .../modules/catalog/catalog-logo-storage.ts | 8 +- .../src/modules/catalog/catalog-manager.ts | 102 ++++------ .../catalog/sync/catalog-sync-domain.ts | 134 +++++++++++-- .../sync/catalog-sync-source-gateway.ts | 182 +++++++++++++++++- .../catalog/sync/catalog-sync-types.ts | 7 +- 9 files changed, 348 insertions(+), 100 deletions(-) diff --git a/opencti-platform/opencti-graphql/src/config/conf.js b/opencti-platform/opencti-graphql/src/config/conf.js index bbd80d288578..f047b2746318 100644 --- a/opencti-platform/opencti-graphql/src/config/conf.js +++ b/opencti-platform/opencti-graphql/src/config/conf.js @@ -542,6 +542,7 @@ export const ENABLED_TRACING = booleanConf('app:telemetry:tracing:enabled', fals export const ENABLED_METRICS = booleanConf('app:telemetry:metrics:enabled', false); export const ENABLED_NOTIFICATION_MANAGER = booleanConf('notification_manager:enabled', true); export const ENABLED_PUBLISHER_MANAGER = booleanConf('publisher_manager:enabled', true); +export const ENABLED_CATALOG_MANAGER = booleanConf('catalog_manager:enabled', true); export const ENABLED_CONNECTOR_MANAGER = booleanConf('connector_manager:enabled', true); export const ENABLED_FILE_INDEX_MANAGER = booleanConf('file_index_manager:enabled', true); diff --git a/opencti-platform/opencti-graphql/src/manager/index.ts b/opencti-platform/opencti-graphql/src/manager/index.ts index 66e1d53ff9a6..ca490ba82c32 100644 --- a/opencti-platform/opencti-graphql/src/manager/index.ts +++ b/opencti-platform/opencti-graphql/src/manager/index.ts @@ -9,3 +9,5 @@ import './retentionManager'; import './telemetryManager'; import './workflowStatusCleanupManager'; import './xtmOneRegistrationManager'; +import './dataSanityManager'; +import '../modules/catalog/catalog-manager'; diff --git a/opencti-platform/opencti-graphql/src/manager/managerModule.ts b/opencti-platform/opencti-graphql/src/manager/managerModule.ts index ff7e21d909ba..ed84656d17cd 100644 --- a/opencti-platform/opencti-graphql/src/manager/managerModule.ts +++ b/opencti-platform/opencti-graphql/src/manager/managerModule.ts @@ -20,6 +20,7 @@ export interface HandlerInput { export interface ManagerCronScheduler { handler: (input?: any) => Promise; shutdown?: () => void; + delayToStart?: number; // Defaults to `interval` interval: number; lockKey: string; infiniteInterval?: number; diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-http.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-http.ts index 61bdc6756c3c..d3a6b5ca2154 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-http.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-http.ts @@ -4,6 +4,7 @@ import { downloadFile } from '../../database/raw-file-storage'; import { logApp } from '../../config/conf'; import { setCookieError } from '../../http/httpUtils'; import { CATALOG_CONTRACT_LOGOS_DIR, getMimeTypeFromImageExtension } from './catalog-logo-storage'; +import { ResourceNotFoundError, UnsupportedError, UnknownError } from '../../config/errors'; export const CATALOG_LOGO_VIEW_PATH = '/catalog/logo/*file'; @@ -14,14 +15,16 @@ export const handleCatalogLogoViewRequest: RequestHandler = async (req, res) => res.sendStatus(403); return; } - const file = req.params.file; + const fileParam = req.params.file; + const file = Array.isArray(fileParam) ? fileParam[0] : fileParam; + logApp.debug('Catalog logo view handler', { file }); if (typeof file !== 'string') { - throw new Error('Invalid URL format'); + throw UnsupportedError('Invalid URL format'); } const s3Key = `${CATALOG_CONTRACT_LOGOS_DIR}/${file}`; const stream = await downloadFile(s3Key); if (!stream) { - const error = new Error('Catalog logo not found'); + const error = ResourceNotFoundError('Catalog logo not found'); logApp.error('Failed to download catalog logo', { cause: error }); res.status(404).send({ status: 'error', error: error.message }); return; @@ -38,7 +41,7 @@ export const handleCatalogLogoViewRequest: RequestHandler = async (req, res) => } stream.pipe(res); } catch (exception) { - const error = exception instanceof Error ? exception : new Error('Unknown error'); + const error = exception instanceof Error ? exception : UnknownError('Unknown error'); setCookieError(res, error.message); logApp.error('Error viewing catalog logo', { cause: error }); res.status(503).send({ status: 'error', error: error.message }); diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-logo-storage.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-logo-storage.ts index e5a596983947..df047d04aa7f 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-logo-storage.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-logo-storage.ts @@ -1,7 +1,7 @@ import crypto from 'crypto'; import { rawListObjects, rawUpload } from '../../database/raw-file-storage'; import { parseDataUrl } from '../../utils/data-url'; -import type { CatalogContractDtoV0 } from './catalog-types'; +import type { CatalogContract } from './catalog-types'; import { CATALOG_LOGO_VIEW_PATH } from './catalog-http'; export const CATALOG_CONTRACT_LOGOS_DIR = 'catalog-logos'; @@ -69,7 +69,7 @@ const computeLogoHash = (logoContent: Buffer) => { }; export const storeCatalogContractLogo = async ( - contractDto: CatalogContractDtoV0, + contractDto: CatalogContract, existingLogos: Set, ) => { const operationResult = computeCatalogContractLogoUploadOperation(contractDto, existingLogos); @@ -102,7 +102,7 @@ export const storeCatalogContractLogo = async ( }; export const computeCatalogContractLogoUploadOperation = ( - contractDto: CatalogContractDtoV0, + contractDto: CatalogContract, existingLogos: Set, ) => { const { logo } = contractDto; @@ -178,7 +178,7 @@ export const computeCatalogContractLogoUploadOperation = ( const extension = getExtensionFromImageMimeType(mimeType); const filename = `${hash}${extension}`; const s3Key = `${CATALOG_CONTRACT_LOGOS_DIR}/${filename}`; - const logoUri = CATALOG_LOGO_VIEW_PATH.replace('*file', s3Key); + const logoUri = CATALOG_LOGO_VIEW_PATH.replace('*file', filename); if (existingLogos.has(filename)) { return { result: 'success' as const, diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-manager.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-manager.ts index 427464f49294..73debdd0e80d 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-manager.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-manager.ts @@ -1,78 +1,46 @@ -import { clearIntervalAsync, setIntervalAsync, type SetIntervalAsyncTimer } from 'set-interval-async/fixed'; -import conf, { booleanConf, logApp } from '../../config/conf'; -import { lockResources } from '../../lock/master-lock'; -import { TYPE_LOCK_ERROR } from '../../config/errors'; +import conf, { booleanConf } from '../../config/conf'; import { executionContext, SYSTEM_USER } from '../../utils/access'; import { autoUpgradeManagedConnectors } from '../connector/connector-domain'; import { synchronizeCatalogs } from './sync/catalog-sync-domain'; +import { registerManager, type ManagerDefinition } from '../../manager/managerModule'; -const CATALOG_MANAGER_ENABLED = booleanConf('app:catalog_manager:enabled', true); -const CATALOG_MANAGER_LOCK_KEY = conf.get('app:catalog_manager:lock_key') || 'catalog_manager_lock'; -const CATALOG_MANAGER_INTERVAL = conf.get('app:catalog_manager:interval'); - -let scheduler: SetIntervalAsyncTimer<[]> | undefined; +const CATALOG_MANAGER_ID = 'CATALOG_MANAGER'; +const CATALOG_MANAGER_LABEL = 'Catalog Manager'; +const CATALOG_MANAGER_CONTEXT = 'catalog_manager'; +const CATALOG_MANAGER_ENABLED = booleanConf('catalog_manager:enabled', true); +const CATALOG_MANAGER_LOCK_KEY = conf.get('catalog_manager:lock_key') ?? 'catalog_manager_lock'; +const CATALOG_MANAGER_INTERVAL = conf.get('catalog_manager:interval') ?? 60_000; export const isCatalogManagerEnabled = () => CATALOG_MANAGER_ENABLED; -const executeScheduledJob = async () => { - let lock; - try { - lock = await lockResources([CATALOG_MANAGER_LOCK_KEY], { retryCount: 0 }); - const context = executionContext('catalog_manager'); - // Sync catalogs to ES - const syncedCatalogsWithChanges = await synchronizeCatalogs(context, SYSTEM_USER); - // Apply upgrade strategy. - // Should be moved another manager maybe (connectorManager). - // Consider using an event to invert the dependency. - await autoUpgradeManagedConnectors(context, SYSTEM_USER, syncedCatalogsWithChanges); - } catch (error: any) { - if (error?.name === TYPE_LOCK_ERROR) { - logApp.debug('[OPENCTI-MODULE] Catalog manager refresh already running on another node'); - return; - } - throw error; - } finally { - if (lock) { - await lock.unlock(); - } - } -}; - -const triggerExecuteScheduledJob = () => { - if (!CATALOG_MANAGER_ENABLED) { - return; - } - void executeScheduledJob().catch((error) => { - logApp.error('[OPENCTI-MODULE] Catalog manager background refresh failed', { cause: error }); - }); -}; - -const start = async () => { - if (!CATALOG_MANAGER_ENABLED) { - logApp.info('[OPENCTI-MODULE] Catalog manager disabled by configuration'); - return; - } - - triggerExecuteScheduledJob(); - - if (CATALOG_MANAGER_INTERVAL && Number(CATALOG_MANAGER_INTERVAL) > 0) { - scheduler = setIntervalAsync(async () => { - triggerExecuteScheduledJob(); - }, Number(CATALOG_MANAGER_INTERVAL)); - logApp.info(`[OPENCTI-MODULE] Catalog manager scheduled every ${Number(CATALOG_MANAGER_INTERVAL)}ms`); - } else { - logApp.info('[OPENCTI-MODULE] Catalog manager configured for startup-only refresh'); - } +const catalogManagerHandler = async () => { + const context = executionContext('catalog_manager'); + // Sync catalogs to ES + const syncedCatalogsWithChanges = await synchronizeCatalogs(context, SYSTEM_USER); + // Apply upgrade strategy. + // Should be moved another manager maybe (connectorManager). + // Consider using an event to invert the dependency. + await autoUpgradeManagedConnectors(context, SYSTEM_USER, syncedCatalogsWithChanges); }; -const shutdown = async () => { - if (scheduler) { - await clearIntervalAsync(scheduler); - scheduler = undefined; - } +const CATALOG_MANAGER_DEFINITION: ManagerDefinition = { + id: CATALOG_MANAGER_ID, + label: CATALOG_MANAGER_LABEL, + executionContext: CATALOG_MANAGER_CONTEXT, + enabledByConfig: CATALOG_MANAGER_ENABLED, + enabled(): boolean { + return this.enabledByConfig && !!CATALOG_MANAGER_LOCK_KEY; + }, + enabledToStart(): boolean { + return this.enabledByConfig && !!CATALOG_MANAGER_LOCK_KEY; + }, + enterpriseEditionOnly: true, + cronSchedulerHandler: { + handler: catalogManagerHandler, + interval: CATALOG_MANAGER_INTERVAL, + lockKey: CATALOG_MANAGER_LOCK_KEY, + }, }; -export default { - start, - shutdown, -}; +// Automatically register manager on start. +registerManager(CATALOG_MANAGER_DEFINITION); diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts b/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts index f45cd242b095..842fb644561f 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts @@ -1,10 +1,11 @@ import { createHash } from 'node:crypto'; -import conf, { logApp } from '../../../config/conf'; +import conf, { isFeatureEnabled, logApp, PLATFORM_VERSION } from '../../../config/conf'; import { deleteCatalogContracts, deleteCatalogs, findCatalogContractsByCatalogId, findCatalogManifestByCatalogId, + findCatalogManifestBySourceUri, findCatalogs, insertCatalogContracts, updateCatalogContracts, @@ -27,12 +28,42 @@ import type { CatalogManifestUpsert, } from '../catalog-types'; import type { CatalogContractSyncSource, CatalogSyncSource, CatalogSyncSourceConfig } from './catalog-sync-types'; -import { fetchSourceCatalog } from './catalog-sync-source-gateway'; +import { fetchSourceCatalog, fetchSourceCatalogRevisionHint } from './catalog-sync-source-gateway'; +import { generateStandardId, idGenFromData } from '../../../schema/identifier'; +import { ENTITY_TYPE_CATALOG_CONTRACT, ENTITY_TYPE_CATALOG_MANIFEST } from '../catalog-types'; +import { UnsupportedError } from '../../../config/errors'; + +const DECOUPLING_VERSIONS_FEATURE_FLAG = 'DECOUPLING_VERSIONS'; + +const getDecouplingRemoteUri = (): string | undefined => { + const xtmHubUrl = conf.get('xtm:xtmhub_url'); + if (!isFeatureEnabled(DECOUPLING_VERSIONS_FEATURE_FLAG) || typeof xtmHubUrl !== 'string' || xtmHubUrl.length === 0) { + return undefined; + } + const normalized = xtmHubUrl.endsWith('/') ? xtmHubUrl.slice(0, -1) : xtmHubUrl; + return `${normalized}/opencti/${encodeURIComponent(PLATFORM_VERSION)}/connector/manifests/latest`; +}; const computeCatalogRevision = (rawManifest: unknown): string => { return createHash('sha256').update(JSON.stringify(rawManifest)).digest('hex'); }; +const buildCatalogContractIds = (catalogId: string, contractId: string) => { + const keyData = { catalog_id: catalogId, contract_id: contractId }; + return { + internal_id: idGenFromData(ENTITY_TYPE_CATALOG_CONTRACT, keyData), + standard_id: generateStandardId(ENTITY_TYPE_CATALOG_CONTRACT, keyData), + }; +}; + +const buildCatalogManifestIds = (catalogId: string) => { + const keyData = { catalog_id: catalogId }; + return { + internal_id: idGenFromData(ENTITY_TYPE_CATALOG_MANIFEST, keyData), + standard_id: generateStandardId(ENTITY_TYPE_CATALOG_MANIFEST, keyData), + }; +}; + export const computeContractContentHash = (contract: CatalogContractSyncSource) => { const replacer = (_key: string, value: Record) => value && typeof value === 'object' && !Array.isArray(value) @@ -54,7 +85,7 @@ const computeCatalogLogosSyncOps = ( for (const sourceContract of sourceCatalog.contracts) { const logoOperationResult = computeCatalogContractLogoUploadOperation(sourceContract, plannedLogos); if (logoOperationResult.result === 'failed') { - throw new Error(`Error while preparing logo for contract ${sourceContract.id}`, { + throw UnsupportedError(`Error while preparing logo for contract ${sourceContract.id}`, { cause: logoOperationResult.error, }); } @@ -65,7 +96,7 @@ const computeCatalogLogosSyncOps = ( plannedLogos.add(logoOperationResult.filename); if (!logoOperationResult.existed) { if (!logoOperationResult.operation) { - throw new Error(`Missing logo upload operation for contract ${sourceContract.id}`); + throw UnsupportedError(`Missing logo upload operation for contract ${sourceContract.id}`); } uploadOperations.push(logoOperationResult.operation); } @@ -103,7 +134,9 @@ const computeCatalogSyncOps = (params: { const sourceContractHash = params.sourceCatalogContractsHashes.get(sourceContract.id)!; const currentContract = params.currentContracts.get(sourceContract.id); if (!currentContract) { + const ids = buildCatalogContractIds(params.sourceCatalog.id, sourceContract.id); const contractCreation: CatalogContractCreation = { + ...ids, catalog_id: params.sourceCatalog.id, contract_id: sourceContract.id, content_hash: sourceContractHash, @@ -122,6 +155,7 @@ const computeCatalogSyncOps = (params: { source_code: sourceContract.source_code ?? undefined, manager_supported: sourceContract.manager_supported, version: sourceContract.container_version, + contract_version: sourceContract.container_version, image: sourceContract.container_image, connector_type: sourceContract.container_type, config_schema: sourceContract.config_schema, @@ -135,7 +169,10 @@ const computeCatalogSyncOps = (params: { if (sourceContractHash === currentContract.content_hash) { return; } + const ids = buildCatalogContractIds(params.sourceCatalog.id, sourceContract.id); const contractUpdate: CatalogContractUpdate = { + internal_id: currentContract.internal_id ?? currentContract.id, + standard_id: ids.standard_id, catalog_id: params.sourceCatalog.id, contract_id: sourceContract.id, content_hash: sourceContractHash, @@ -153,6 +190,7 @@ const computeCatalogSyncOps = (params: { subscription_link: sourceContract.subscription_link, manager_supported: sourceContract.manager_supported, version: sourceContract.container_version, + contract_version: sourceContract.container_version, image: sourceContract.container_image, connector_type: sourceContract.container_type, config_schema: sourceContract.config_schema, @@ -170,7 +208,9 @@ const computeCatalogSyncOps = (params: { return; } }); + const catalogManifestIds = buildCatalogManifestIds(params.sourceCatalog.id); const catalogManifestUpsert: CatalogManifestUpsert = { + ...catalogManifestIds, revision: params.revision, source_uri: params.sourceConfig.uri, catalog_id: params.sourceCatalog.id, @@ -188,6 +228,30 @@ const computeCatalogSyncOps = (params: { const synchronizeCatalog = async (context: AuthContext, user: AuthUser, sourceConfig: CatalogSyncSourceConfig) => { try { + logApp.debug('[OPENCTI-MODULE] Synchronizing catalog', { + sourceKind: sourceConfig.kind, + sourceUri: sourceConfig.uri, + }); + const currentCatalogBySourceUri = sourceConfig.kind === 'remote' + ? await findCatalogManifestBySourceUri(context, user, sourceConfig.uri) + : undefined; + const remoteRevisionHint = sourceConfig.kind === 'remote' + ? await fetchSourceCatalogRevisionHint(sourceConfig) + : undefined; + if (sourceConfig.kind === 'remote' && currentCatalogBySourceUri?.revision) { + if (remoteRevisionHint && remoteRevisionHint === currentCatalogBySourceUri.revision) { + logApp.info('[OPENCTI-MODULE] Catalog manager manifest unchanged (remote ETag match)', { + sourceKind: sourceConfig.kind, + catalogId: currentCatalogBySourceUri.catalog_id, + revision: remoteRevisionHint, + module: 'catalog', + }); + return { + synced: false, + catalogId: currentCatalogBySourceUri.catalog_id, + } as const; + } + } // Fetch source catalog const sourceCatalog = await fetchSourceCatalog(sourceConfig); // Find existing persisted data in the database corresponding to the catalog id @@ -206,16 +270,16 @@ const synchronizeCatalog = async (context: AuthContext, user: AuthUser, sourceCo module: 'catalog', }); } - if (currentCatalog && currentCatalog.id !== sourceCatalog.id) { + if (currentCatalogBySourceUri && currentCatalogBySourceUri.catalog_id !== sourceCatalog.id) { logApp.warn('[OPENCTI-MODULE] Same catalog source with different ID', { sourceKind: sourceConfig.kind, - knownId: currentCatalog.id, + knownId: currentCatalogBySourceUri.catalog_id, newId: sourceCatalog.id, module: 'catalog', }); } // Compute revision and compare with persisted, return early if equal. - const revision = computeCatalogRevision(sourceCatalog); + const revision = remoteRevisionHint ?? computeCatalogRevision(sourceCatalog); if (currentRevision && revision === currentRevision) { logApp.info('[OPENCTI-MODULE] Catalog manager manifest unchanged (revision match)', { sourceKind: sourceConfig.kind, @@ -230,7 +294,7 @@ const synchronizeCatalog = async (context: AuthContext, user: AuthUser, sourceCo } // Fetch all current contracts, logos & compute sync diff const currentContracts = currentCatalog - ? await findCatalogContractsByCatalogId(context, user, currentCatalog.id) + ? await findCatalogContractsByCatalogId(context, user, currentCatalog.catalog_id) : new Map(); const currentLogos = await listCatalogContractLogos(); const logoSyncOps = computeCatalogLogosSyncOps(sourceCatalog, currentLogos); @@ -248,6 +312,14 @@ const synchronizeCatalog = async (context: AuthContext, user: AuthUser, sourceCo currentContracts, }); // Persist refreshed catalog, contracts & logos + // Something's fishy if this occurs too frequently. + logApp.warn('[OPENCTI-MODULE] Persisting catalog & contracts', { + catalogId: sourceCatalog.id, + contractsCreationsCount: catalogSyncDiff.contractsCreations.length, + contractsUpdatesCount: catalogSyncDiff.contractsUpdates.length, + contractsDeletionsCount: catalogSyncDiff.contractsDeletions.length, + module: 'catalog', + }); await uploadCatalogContractLogos(logoSyncOps.uploadOperations); await insertCatalogContracts(context, user, catalogSyncDiff.contractsCreations); if (catalogSyncDiff.contractsUpdates.length > 0) { @@ -280,19 +352,32 @@ const synchronizeCatalog = async (context: AuthContext, user: AuthUser, sourceCo } }; +const EMBEDDED_CATALOG_SYNC_SOURCE: CatalogSyncSourceConfig = { kind: 'embedded', uri: 'embedded' } as const; + const initSyncSources = () => { // Build catalog sources configs from env vars/app settings - const sources: CatalogSyncSourceConfig[] = [{ - kind: 'embedded' as const, - uri: 'embedded', - }]; + const sources: CatalogSyncSourceConfig[] = []; + const decouplingRemoteUri = getDecouplingRemoteUri(); + if (decouplingRemoteUri) { + sources.push({ kind: 'remote', uri: decouplingRemoteUri }); + } else { + sources.push(EMBEDDED_CATALOG_SYNC_SOURCE); + } const CUSTOM_CATALOGS: string[] = conf.get('app:custom_catalogs') ?? []; if (CUSTOM_CATALOGS) { - sources.push(...CUSTOM_CATALOGS.map((customCatalog) => ({ - kind: 'local' as const, - filepath: customCatalog, - uri: 'file:///' + customCatalog, // TODO: normalize - }))); + sources.push(...CUSTOM_CATALOGS.map((customCatalog) => { + if (customCatalog.startsWith('http://') || customCatalog.startsWith('https://')) { + return { + kind: 'remote' as const, + uri: customCatalog, + }; + } + return { + kind: 'local' as const, + filepath: customCatalog, + uri: 'file:///' + customCatalog, // TODO: normalize + }; + })); } return sources; }; @@ -311,15 +396,28 @@ const cleanupObsoleteCatalogs = async (context: AuthContext, syncedCatalogs: str export const synchronizeCatalogs = async (context: AuthContext, user: AuthUser) => { const sources = initSyncSources(); + const decouplingRemoteUri = getDecouplingRemoteUri(); logApp.debug('[OPENCTI-MODULE] Synchronizing catalogs', { count: sources.length, + decouplingRemoteUri, module: 'catalog', }); const syncedCatalogs: string[] = []; const syncedCatalogsWithChanges: string[] = []; // Sync catalogs from sources for (const source of sources) { - const result = await synchronizeCatalog(context, user, source); + let result = await synchronizeCatalog(context, user, source); + if (source.kind === 'remote' && result.error && decouplingRemoteUri && source.uri === decouplingRemoteUri) { + const alreadyPersistedRemoteCatalog = await findCatalogManifestBySourceUri(context, user, source.uri); + if (!alreadyPersistedRemoteCatalog) { + logApp.error('[OPENCTI-MODULE] Remote catalog source failed before first successful persistence, falling back to embedded source', { + sourceKind: source.kind, + sourceUri: source.uri, + module: 'catalog', + }); + result = await synchronizeCatalog(context, user, EMBEDDED_CATALOG_SYNC_SOURCE); + } + } if (!result.error) { syncedCatalogs.push(result.catalogId); if (result.synced) { diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-source-gateway.ts b/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-source-gateway.ts index e562e53c4d25..3e105cdc7f6a 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-source-gateway.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-source-gateway.ts @@ -1,15 +1,71 @@ import { readFile } from 'node:fs/promises'; -import type { CatalogContractDtoV0, CatalogDefinition, CatalogDtoV0 } from '../catalog-types'; +import type { CatalogContract, CatalogDefinition, TypedProperty } from '../catalog-types'; import type { CatalogContractSyncSource, CatalogSyncSource, CatalogSyncSourceConfig } from './catalog-sync-types'; -import { logApp } from '../../../config/conf'; +import conf, { logApp } from '../../../config/conf'; import { isEmptyField } from '../../../database/utils'; import { UnsupportedError } from '../../../config/errors'; import { getOrCompileValidator } from '../catalog-domain'; +import { getHttpClient } from '../../../utils/http-client'; + +type CatalogContractDtoV0 = CatalogContract; +type CatalogDtoV0 = CatalogDefinition; +type CatalogContractDtoV1 = { + id: string; + title: string; + slug: string; + description: string; + short_description: string; + logo: string | null; + use_cases: string[]; + verified: boolean; + last_verified_date: string; + subscription_link: string | null; + source_code: string | null; + manager_supported: boolean; + support_version: string | null; + license_type: 'Free' | 'Commercial' | null; + contact: string | null; + solution_categories: string[]; + version: string | null; + image_name: string | null; + image_type: string; + additional_properties: Record; + config_schema: Record; +}; +type CatalogDtoV1 = { + id: string; + name: string; + description: string; + manifest_schema_version: '1'; + manifest_version: string; + product_version: string; + contracts: Array; +}; interface CatalogSyncSourceAdapter { - fetch: () => Promise; + fetch: () => Promise; + fetchRevisionHint?: () => Promise; } +const DEFAULT_REMOTE_CATALOG_TIMEOUT_MS = 30000; +const resolveRemoteCatalogTimeoutMs = () => { + const configuredTimeout = Number(conf.get('app:catalog_sync_remote_timeout') ?? conf.get('app:request_timeout')); + if (!Number.isFinite(configuredTimeout) || configuredTimeout <= 0) { + return DEFAULT_REMOTE_CATALOG_TIMEOUT_MS; + } + return configuredTimeout; +}; + +const withAbortTimeout = async (timeoutMs: number, call: (signal: AbortSignal) => Promise): Promise => { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + try { + return await call(controller.signal); + } finally { + clearTimeout(timer); + } +}; + class EmbeddedCatalogSyncSource implements CatalogSyncSourceAdapter { constructor(private _sourceConfig: Extract) {} @@ -28,6 +84,29 @@ class LocalCatalogSyncSource implements CatalogSyncSourceAdapter { } } +class RemoteCatalogSyncSource implements CatalogSyncSourceAdapter { + constructor(private sourceConfig: Extract) {} + + async fetch() { + const timeout = resolveRemoteCatalogTimeoutMs(); + const client = getHttpClient({ responseType: 'text', timeout }); + const response = await withAbortTimeout(timeout, (signal) => client.get(this.sourceConfig.uri, { signal })); + return JSON.parse(response.data); + } + + async fetchRevisionHint() { + const timeout = resolveRemoteCatalogTimeoutMs(); + const client = getHttpClient({ responseType: 'text', timeout }); + const response = await withAbortTimeout(timeout, (signal) => client.head(this.sourceConfig.uri, { signal })); + const etagHeader = response.headers?.etag; + const etag = Array.isArray(etagHeader) ? etagHeader[0] : etagHeader; + if (!etag || typeof etag !== 'string') { + return undefined; + } + return etag; + } +} + const getCatalogSourceAdapter = (config: CatalogSyncSourceConfig): CatalogSyncSourceAdapter => { switch (config.kind) { case 'local': { @@ -36,8 +115,11 @@ const getCatalogSourceAdapter = (config: CatalogSyncSourceConfig): CatalogSyncSo case 'embedded': { return new EmbeddedCatalogSyncSource(config); } + case 'remote': { + return new RemoteCatalogSyncSource(config); + } default: - throw new Error(`Unknown catalog sync source kind (${JSON.stringify(config)})`); + throw UnsupportedError(`Unknown catalog sync source kind (${JSON.stringify(config)})`); }; }; @@ -74,16 +156,101 @@ const validateSyncSource = (syncSource: CatalogSyncSource) => { } }; +const DEFAULT_CONFIG_SCHEMA: CatalogContractDtoV0['config_schema'] = { + $schema: 'https://json-schema.org/draft/2020-12/schema', + $id: '', + type: 'object', + properties: {}, + required: [], + additionalProperties: true, +}; + +const normalizeConfigSchema = (configSchema: unknown): CatalogContractDtoV0['config_schema'] => { + if (!configSchema || typeof configSchema !== 'object' || Array.isArray(configSchema)) { + return DEFAULT_CONFIG_SCHEMA; + } + const schema = configSchema as Record; + return { + $schema: typeof schema.$schema === 'string' ? schema.$schema : DEFAULT_CONFIG_SCHEMA.$schema, + $id: typeof schema.$id === 'string' ? schema.$id : DEFAULT_CONFIG_SCHEMA.$id, + type: typeof schema.type === 'string' ? schema.type : DEFAULT_CONFIG_SCHEMA.type, + properties: schema.properties && typeof schema.properties === 'object' && !Array.isArray(schema.properties) + ? schema.properties as Record + : DEFAULT_CONFIG_SCHEMA.properties, + required: Array.isArray(schema.required) + ? schema.required.filter((item): item is string => typeof item === 'string') + : DEFAULT_CONFIG_SCHEMA.required, + additionalProperties: typeof schema.additionalProperties === 'boolean' + ? schema.additionalProperties + : DEFAULT_CONFIG_SCHEMA.additionalProperties, + }; +}; + +const normalizeSupportVersion = (supportVersion: string | null | undefined): string | null => { + if (!supportVersion) { + return null; + } + const normalized = supportVersion.replace(/^\s*>=\s*/, '').trim(); + return normalized.length > 0 ? normalized : null; +}; + +const mapCatalogContractDtoV1ToCatalogContractDtoV0 = (contractDto: CatalogContractDtoV1): CatalogContractDtoV0 => { + return { + title: contractDto.title, + slug: contractDto.slug, + description: contractDto.description, + short_description: contractDto.short_description, + logo: contractDto.logo, + use_cases: contractDto.use_cases, + verified: contractDto.verified, + last_verified_date: contractDto.last_verified_date, + playbook_supported: false, + max_confidence_level: 100, + support_version: normalizeSupportVersion(contractDto.support_version), + subscription_link: contractDto.subscription_link, + source_code: contractDto.source_code ?? '', + manager_supported: contractDto.manager_supported, + container_version: contractDto.version ?? '', + container_image: contractDto.image_name ?? '', + container_type: contractDto.image_type, + config_schema: normalizeConfigSchema(contractDto.config_schema), + license_type: contractDto.license_type, + solution_categories: contractDto.solution_categories, + contact: contractDto.contact, + }; +}; + +const isCatalogDtoV1 = (catalog: CatalogDtoV0 | CatalogDtoV1): catalog is CatalogDtoV1 => { + return typeof (catalog as CatalogDtoV1).manifest_schema_version === 'string'; +}; + export const mapCatalogContractDtoToCatalogContractSyncSource = ( contractDto: CatalogContractDtoV0, ): CatalogContractSyncSource => { + const normalizedSupportVersion = normalizeSupportVersion(contractDto.support_version); return { id: `${contractDto.slug}-${contractDto.container_version}`, ...contractDto, + support_version: normalizedSupportVersion, }; }; -export const mapCatalogDtoToCatalogSyncSource = (catalog: CatalogDtoV0) => { +export const mapCatalogDtoToCatalogSyncSource = (catalog: CatalogDtoV0 | CatalogDtoV1) => { + if (isCatalogDtoV1(catalog)) { + return { + id: catalog.id, + name: catalog.name, + description: catalog.description, + version: catalog.product_version, + contracts: catalog.contracts.map((contractDto) => { + const normalizedContractDto = mapCatalogContractDtoV1ToCatalogContractDtoV0(contractDto); + return { + id: contractDto.id, + ...normalizedContractDto, + }; + }), + }; + } const syncSource = { ...catalog, contracts: catalog.contracts.map(mapCatalogContractDtoToCatalogContractSyncSource), @@ -98,3 +265,8 @@ export const fetchSourceCatalog = async (sourceConfig: CatalogSyncSourceConfig): validateSyncSource(syncSource); return syncSource; }; + +export const fetchSourceCatalogRevisionHint = async (sourceConfig: CatalogSyncSourceConfig): Promise => { + const adapter = getCatalogSourceAdapter(sourceConfig); + return adapter.fetchRevisionHint?.(); +}; diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-types.ts b/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-types.ts index 21d244afad45..a5e91c8dece2 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-types.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-types.ts @@ -1,4 +1,4 @@ -import type { CatalogContractDtoV0 } from '../catalog-types'; +import type { CatalogContract } from '../catalog-types'; export type CatalogSyncSourceConfig = { kind: 'embedded'; @@ -7,9 +7,12 @@ export type CatalogSyncSourceConfig = { kind: 'local'; filepath: string; uri: string; +} | { + kind: 'remote'; + uri: string; }; -export type CatalogContractSyncSource = CatalogContractDtoV0 & { +export type CatalogContractSyncSource = CatalogContract & { id: string; // slug-version }; From 0690e7c5d22199e3bd640995e40e9db9d4a02e54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Tue, 18 Aug 2026 20:58:13 +0200 Subject: [PATCH 05/45] feat(catalog): add version-aware contract compatibility selection --- .../src/utils/filters/filtersUtils.tsx | 4 +- .../src/database/engine-mapping-generator.ts | 4 + .../opencti-graphql/src/database/engine.ts | 2 +- .../src/domain/filterKeysSchema.ts | 6 +- .../src/modules/catalog/catalog-repository.ts | 158 ++++++++++++++++-- .../src/modules/catalog/catalog-types.ts | 24 ++- .../src/modules/catalog/catalog.graphql | 5 + .../src/modules/catalog/catalog.ts | 16 +- .../src/schema/attribute-definition.ts | 5 +- .../src/schema/schema-attributes.ts | 21 +-- .../opencti-graphql/src/utils/sorting.ts | 8 +- 11 files changed, 199 insertions(+), 54 deletions(-) diff --git a/opencti-platform/opencti-front/src/utils/filters/filtersUtils.tsx b/opencti-platform/opencti-front/src/utils/filters/filtersUtils.tsx index 4d5d31288f6a..33401d01d57d 100644 --- a/opencti-platform/opencti-front/src/utils/filters/filtersUtils.tsx +++ b/opencti-platform/opencti-front/src/utils/filters/filtersUtils.tsx @@ -222,7 +222,7 @@ export const isFilterGroupFormatCorrect = (value: unknown): boolean => { export const isUniqFilter = (key: string, filterKeysSchema: Map>) => { const filterDefinition = filterKeysSchema.get('Stix-Core-Object')?.get(key); - return !!(filterDefinition && ['boolean', 'date', 'integer', 'float'].includes(filterDefinition.type)); + return !!(filterDefinition && ['boolean', 'date', 'integer', 'float', 'version'].includes(filterDefinition.type)); }; // basic text filters are filters of type string or text that are not entity types filters @@ -238,7 +238,7 @@ export const isBasicTextFilter = ( export const isNumericFilter = ( filterType?: string, ) => { - return filterType === 'integer' || filterType === 'float'; + return filterType === 'integer' || filterType === 'float' || filterType === 'version'; }; /** diff --git a/opencti-platform/opencti-graphql/src/database/engine-mapping-generator.ts b/opencti-platform/opencti-graphql/src/database/engine-mapping-generator.ts index 3c2c745aba04..07a8c2a2093a 100644 --- a/opencti-platform/opencti-graphql/src/database/engine-mapping-generator.ts +++ b/opencti-platform/opencti-graphql/src/database/engine-mapping-generator.ts @@ -11,6 +11,7 @@ import { shortMapping, shortStringFormats, textMapping, + versionMapping, } from '../schema/attribute-definition'; import { ABSTRACT_STIX_REF_RELATIONSHIP } from '../schema/general'; import { INTERNAL_RELATIONSHIPS } from '../schema/internalRelationship'; @@ -26,6 +27,9 @@ export const engineMappingGenerator = (engine: ElkClient | OpenClient): Record { if (entityAttribute.type === 'string') { + if (entityAttribute.format === 'version') { + return versionMapping; + } if (shortStringFormats.includes(entityAttribute.format)) { return shortMapping; } diff --git a/opencti-platform/opencti-graphql/src/database/engine.ts b/opencti-platform/opencti-graphql/src/database/engine.ts index 12b1e1c052c4..0bc61817bfbd 100644 --- a/opencti-platform/opencti-graphql/src/database/engine.ts +++ b/opencti-platform/opencti-graphql/src/database/engine.ts @@ -3205,7 +3205,7 @@ export const elLoadBy = async ( user: AuthUser, field: string, value: any, - type = null, + type: string | null = null, indices: string[] = READ_DATA_INDICES, ) => { const filters = { diff --git a/opencti-platform/opencti-graphql/src/domain/filterKeysSchema.ts b/opencti-platform/opencti-graphql/src/domain/filterKeysSchema.ts index 772bc777e99f..7188a6302567 100644 --- a/opencti-platform/opencti-graphql/src/domain/filterKeysSchema.ts +++ b/opencti-platform/opencti-graphql/src/domain/filterKeysSchema.ts @@ -63,7 +63,7 @@ import { CUSTOM_FIELDS_FEATURE_FLAG, isFeatureEnabled } from '../config/conf'; export type FilterDefinition = { filterKey: string; - type: string; // possible values: boolean, date, integer, float, string, id, vocabulary, text, enum, object, nested + type: string; // possible values: boolean, date, integer, float, version, string, id, vocabulary, text, enum, object, nested label: string; // filter key translation in English multiple: boolean; // if the field can have multiple values subEntityTypes: string[]; // entity types that have the given type as parent and have this filter key in their schema @@ -89,6 +89,8 @@ const buildFilterDefinitionFromAttributeDefinition = (attributeDefinition: Attri if ((attributeDefinition as StringAttribute).format === 'id') { type = 'id'; elementsForFilterValuesSearch = (attributeDefinition as IdAttribute).entityTypes; + } else if ((attributeDefinition as StringAttribute).format === 'version') { + type = 'version'; } else if ((attributeDefinition as StringAttribute).format === 'short') { type = 'string'; } else if ((attributeDefinition as StringAttribute).format === 'vocabulary') { @@ -100,7 +102,7 @@ const buildFilterDefinitionFromAttributeDefinition = (attributeDefinition: Attri type = 'enum'; elementsForFilterValuesSearch = (attributeDefinition as EnumAttribute).values; } else { - throw Error(`A string attribute definition format can be 'id', 'short', 'text' or 'json', but not ${attributeDefinition.format}`); + throw Error(`A string attribute definition format can be 'id', 'short', 'version', 'text' or 'json', but not ${attributeDefinition.format}`); } } // return the filter definition diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts index d6303b3e8d64..8d0d20ca3ea9 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts @@ -1,7 +1,8 @@ +import { PLATFORM_VERSION } from '../../config/conf'; import { elDeleteInstances, elIndex, elIndexElements, elLoadBy } from '../../database/engine'; import { fullEntitiesList } from '../../database/middleware-loader'; -import { INDEX_INTERNAL_OBJECTS } from '../../database/utils'; -import { FilterMode, FilterOperator } from '../../generated/graphql'; +import { INDEX_INTERNAL_OBJECTS, READ_INDEX_INTERNAL_OBJECTS } from '../../database/utils'; +import { FilterMode, FilterOperator, OrderingMode } from '../../generated/graphql'; import type { BasicStoreBase } from '../../types/store'; import type { AuthContext, AuthUser } from '../../types/user'; import { @@ -24,7 +25,16 @@ export const findCatalogManifestByCatalogId = async ( user: AuthUser, catalogId: string, ) => { - const catalog = await elLoadBy(context, user, 'catalog_id', catalogId); + const catalog = await elLoadBy(context, user, 'catalog_id', catalogId, ENTITY_TYPE_CATALOG_MANIFEST); + return catalog; +}; + +export const findCatalogManifestBySourceUri = async ( + context: AuthContext, + user: AuthUser, + sourceUri: string, +) => { + const catalog = await elLoadBy(context, user, 'source_uri', sourceUri); return catalog; }; @@ -49,7 +59,7 @@ export const findCatalogs = async (context: AuthContext, user: AuthUser, exclude context, user, [ENTITY_TYPE_CATALOG_MANIFEST], - { filters }, + { indices: [READ_INDEX_INTERNAL_OBJECTS], filters }, ); return catalogs; }; @@ -72,6 +82,7 @@ export const findCatalogContractsByCatalogId = async ( user, [ENTITY_TYPE_CATALOG_CONTRACT], { + indices: [READ_INDEX_INTERNAL_OBJECTS], filters: { filters: [{ key: ['catalog_id'], @@ -82,13 +93,70 @@ export const findCatalogContractsByCatalogId = async ( }, }, ); - // TODO: check if filter with support_version return contracts.reduce((map, contract) => { map.set(contract.contract_id, contract); return map; }, new Map()); }; +export const findLatestCompatibleCatalogContractsByCatalogId = async ( + context: AuthContext, + user: AuthUser, + catalogId: string, +) => { + const contracts = await fullEntitiesList( + context, + user, + [ENTITY_TYPE_CATALOG_CONTRACT], + { + indices: [READ_INDEX_INTERNAL_OBJECTS], + filters: { + filters: [{ + key: ['catalog_id'], + values: [catalogId], + }], + filterGroups: [ + { + mode: FilterMode.Or, + filters: [], + filterGroups: [ + { + mode: FilterMode.And, + filters: [{ + key: ['support_version'], + values: [PLATFORM_VERSION], + operator: FilterOperator.Lte, + }], + filterGroups: [], + }, + { + mode: FilterMode.And, + filters: [{ + key: ['support_version'], + values: ['EXISTS'], + operator: FilterOperator.NotEq, + }], + filterGroups: [], + }, + ], + }, + ], + mode: FilterMode.And, + }, + orderBy: 'contract_version', + orderMode: OrderingMode.Desc, + }, + ); + // Keep latest compatible version by slug (results are sorted by version desc). + return contracts.reduce((map, contract) => { + if (map.has(contract.slug)) { + return map; + } + map.set(contract.slug, contract); + return map; + }, new Map()); +}; + export const findLatestCompatibleCatalogContractBySlug = async ( context: AuthContext, user: AuthUser, @@ -99,18 +167,45 @@ export const findLatestCompatibleCatalogContractBySlug = async ( user, [ENTITY_TYPE_CATALOG_CONTRACT], { + indices: [READ_INDEX_INTERNAL_OBJECTS], filters: { filters: [{ key: ['slug'], values: [contractSlug], }], - filterGroups: [], + filterGroups: [ + { + mode: FilterMode.Or, + filters: [], + filterGroups: [ + { + mode: FilterMode.And, + filters: [{ + key: ['support_version'], + values: [PLATFORM_VERSION], + operator: FilterOperator.Lte, + }], + filterGroups: [], + }, + { + mode: FilterMode.And, + filters: [{ + key: ['support_version'], + values: ['EXISTS'], + operator: FilterOperator.NotEq, + }], + filterGroups: [], + }, + ], + }, + ], mode: FilterMode.And, }, + orderBy: 'contract_version', + orderMode: OrderingMode.Desc, + first: 1, }, ); - // Filter with support_version - // Sort by version DESC return contracts[0]; }; @@ -124,18 +219,45 @@ export const findLatestCompatibleCatalogContractByImageName = async ( user, [ENTITY_TYPE_CATALOG_CONTRACT], { + indices: [READ_INDEX_INTERNAL_OBJECTS], filters: { filters: [{ key: ['image'], values: [imageName], }], - filterGroups: [], + filterGroups: [ + { + mode: FilterMode.Or, + filters: [], + filterGroups: [ + { + mode: FilterMode.And, + filters: [{ + key: ['support_version'], + values: [PLATFORM_VERSION], + operator: FilterOperator.Lte, + }], + filterGroups: [], + }, + { + mode: FilterMode.And, + filters: [{ + key: ['support_version'], + values: ['EXISTS'], + operator: FilterOperator.NotEq, + }], + filterGroups: [], + }, + ], + }, + ], mode: FilterMode.And, }, + orderBy: 'contract_version', + orderMode: OrderingMode.Desc, + first: 1, }, ); - // Filter with support_version - // Sort by version DESC return contracts[0]; }; @@ -144,7 +266,12 @@ export const insertCatalogContracts = async ( user: AuthUser, contracts: CatalogContractCreation[], ) => { - await elIndexElements(context, user, ENTITY_TYPE_CATALOG_CONTRACT, contracts); + const contractsToIndex = contracts.map((contract) => ({ + ...contract, + _index: INDEX_INTERNAL_OBJECTS, + entity_type: ENTITY_TYPE_CATALOG_CONTRACT, + })); + await elIndexElements(context, user, ENTITY_TYPE_CATALOG_CONTRACT, contractsToIndex); }; export const updateCatalogContracts = async ( @@ -153,7 +280,12 @@ export const updateCatalogContracts = async ( updates: CatalogContractUpdate[], ) => { // We can use bulk `index` as we provide the entire documents - await elIndexElements(context, user, ENTITY_TYPE_CATALOG_CONTRACT, updates); + const contractsToIndex = updates.map((update) => ({ + ...update, + _index: INDEX_INTERNAL_OBJECTS, + entity_type: ENTITY_TYPE_CATALOG_CONTRACT, + })); + await elIndexElements(context, user, ENTITY_TYPE_CATALOG_CONTRACT, contractsToIndex); }; export const deleteCatalogContracts = async ( diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-types.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-types.ts index 8491004c3619..6cd8177758d9 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-types.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-types.ts @@ -1,7 +1,7 @@ import type { BasicStoreEntity, StoreEntity } from '../../types/store'; // region DTO types -export type IngestionConnectorType = 'INTERNAL_ENRICHMENT' | 'EXTERNAL_IMPORT' | 'INTERNAL_EXPORT_FILE' | 'INTERNAL_IMPORT_FILE'; +export type IngestionConnectorType = string; type TypeMap = { string: string; @@ -11,14 +11,14 @@ type TypeMap = { boolean: boolean; }; -type TypedProperty = { +export type TypedProperty = { type: K; default?: TypeMap[K]; description?: string; format?: string; }; -export interface CatalogContractDtoV0 { +export interface CatalogContract { title: string; slug: string; description: string; @@ -51,27 +51,22 @@ export interface CatalogContractDtoV0 { contact: string | null; } -export interface CatalogDtoV0 { +export interface CatalogDefinition { id: string; name: string; description: string; version: string; - contracts: Array; + contracts: Array; } // endregion -// region Domain types -export type CatalogContract = CatalogContractDtoV0; -export type CatalogDefinition = CatalogDtoV0; -// endregion - export interface CatalogType { definition: CatalogDefinition; graphql: GraphqlCatalog; } // region Api types -export type GraphqlCatalogContract = CatalogContractDtoV0; +export type GraphqlCatalogContract = CatalogContract; export interface GraphqlCatalog { id: string; entity_type: string; @@ -111,6 +106,7 @@ export interface CatalogContractEntityFields { source_code?: string; manager_supported: boolean; version: string; + contract_version: string; image: string; connector_type: IngestionConnectorType; config_schema: { @@ -151,9 +147,9 @@ export interface CatalogContractDeletion { export type CatalogContractUpdate = Required<{ [TKey in keyof CatalogContractEntityFields]: CatalogContractEntityFields[TKey] | null; -}>; +}> & { internal_id: string; standard_id: string }; -export type CatalogContractCreation = CatalogContractEntityFields; +export type CatalogContractCreation = CatalogContractEntityFields & { internal_id: string; standard_id: string }; -export type CatalogManifestUpsert = CatalogManifestEntityFields; +export type CatalogManifestUpsert = CatalogManifestEntityFields & { internal_id: string; standard_id: string }; // endregion diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog.graphql b/opencti-platform/opencti-graphql/src/modules/catalog/catalog.graphql index 54595f010340..651644bd3ace 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog.graphql +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog.graphql @@ -34,7 +34,12 @@ type CatalogEdge { # Queries type Query { + # Returns only the latest compatible version of each contract for + # backwards compatibility. catalog(id: String!): Catalog @auth(for: [INGESTION]) + # Returns only the latest compatible version of each contract for + # backwards compatibility. catalogs: [Catalog!]! @auth(for: [INGESTION]) + # Returns only the latest compatible version for backwards compatibility. contract(slug: String!): ExtendedContract @auth(for: [INGESTION]) } diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog.ts index b69e82625207..5f38f7ea5c0c 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog.ts @@ -9,6 +9,7 @@ import { type StoreEntityCatalogManifest, } from './catalog-types'; import type { AttributeDefinition } from '../../schema/attribute-definition'; +import { UnsupportedError } from '../../config/errors'; export const CATALOG_CONTRACT_MAPPINGS: AttributeDefinition[] = [ { name: 'catalog_id', label: 'Catalog ID', type: 'string', format: 'short', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: true }, @@ -24,11 +25,12 @@ export const CATALOG_CONTRACT_MAPPINGS: AttributeDefinition[] = [ { name: 'last_verified_date', label: 'Last verified date', type: 'string', format: 'short', mandatoryType: 'no', editDefault: false, multiple: false, upsert: false, isFilterable: false }, { name: 'playbook_supported', label: 'Playbook supported', type: 'boolean', mandatoryType: 'no', editDefault: false, multiple: false, upsert: false, isFilterable: true }, { name: 'max_confidence_level', label: 'Max confidence level', type: 'numeric', precision: 'integer', mandatoryType: 'no', editDefault: false, multiple: false, upsert: false, isFilterable: false }, - { name: 'support_version', label: 'Support version', type: 'string', format: 'short', mandatoryType: 'no', editDefault: false, multiple: false, upsert: false, isFilterable: false }, + { name: 'support_version', label: 'Support version', type: 'string', format: 'version', mandatoryType: 'no', editDefault: false, multiple: false, upsert: false, isFilterable: false }, { name: 'subscription_link', label: 'Subscription link', type: 'string', format: 'short', mandatoryType: 'no', editDefault: false, multiple: false, upsert: false, isFilterable: false }, { name: 'source_code', label: 'Source code', type: 'string', format: 'short', mandatoryType: 'no', editDefault: false, multiple: false, upsert: false, isFilterable: false }, { name: 'manager_supported', label: 'Manager supported', type: 'boolean', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: true }, { name: 'version', label: 'Version', type: 'string', format: 'short', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: true }, + { name: 'contract_version', label: 'Contract version', type: 'string', format: 'version', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: false }, { name: 'image', label: 'Image', type: 'string', format: 'short', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: true }, { name: 'connector_type', label: 'Connector type', type: 'string', format: 'short', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: true }, { name: 'config_schema', label: 'Configuration schema', type: 'object', format: 'raw', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: false }, @@ -47,7 +49,9 @@ const CATALOG_CONTRACT_DEFINITION: ModuleDefinition { - throw new Error('Should not be called'); + throw UnsupportedError('Catalog contract representative should not be called'); }, converter_2_1: () => { - throw new Error('Should not be called'); + throw UnsupportedError('Catalog contract converter_2_1 should not be called'); }, }; @@ -87,10 +91,10 @@ const CATALOG_MANIFEST_DEFINITION: ModuleDefinition { - throw new Error('Should not be called'); + throw UnsupportedError('Catalog manifest representative should not be called'); }, converter_2_1: () => { - throw new Error('Should not be called'); + throw UnsupportedError('Catalog manifest converter_2_1 should not be called'); }, }; diff --git a/opencti-platform/opencti-graphql/src/schema/attribute-definition.ts b/opencti-platform/opencti-graphql/src/schema/attribute-definition.ts index e38a8e2b14b7..2311ea33745b 100644 --- a/opencti-platform/opencti-graphql/src/schema/attribute-definition.ts +++ b/opencti-platform/opencti-graphql/src/schema/attribute-definition.ts @@ -20,6 +20,7 @@ export const shortMapping = { }, }; export const textMapping = { type: 'text' }; +export const versionMapping = { type: 'version' }; export const dateMapping = { type: 'date' }; export const booleanMapping = { type: 'boolean' }; export const numericMapping = (precision: string) => ({ type: precision, coerce: false }); @@ -76,7 +77,7 @@ export type DateAttribute = { type: 'date' } & BasicDefinition; export type BooleanAttribute = { type: 'boolean' } & BasicDefinition; export type NumericAttribute = { type: 'numeric'; precision: 'integer' | 'long' | 'float'; scalable?: boolean } & BasicDefinition; export type IdAttribute = { type: 'string'; format: 'id'; entityTypes: string[]; attrRawIds?: GetRawIdsFn; representative?: RepresentativeFn } & BasicDefinition; -export type TextAttribute = { type: 'string'; format: 'short' | 'text' } & BasicDefinition; +export type TextAttribute = { type: 'string'; format: 'short' | 'text' | 'version' } & BasicDefinition; export type EnumAttribute = { type: 'string'; format: 'enum'; values: string[] } & BasicDefinition; export type VocabAttribute = { type: 'string'; format: 'vocabulary'; vocabularyCategory: string } & BasicDefinition; export type JsonAttribute = { type: 'string'; format: 'json'; attrRawIds?: GetRawIdsFn; representative?: RepresentativeFn; multiple: false; schemaDef?: Record } & BasicDefinition; @@ -93,7 +94,7 @@ export type ComplexAttributeWithMappings = NumericAttribute | DateAttribute | BooleanAttribute | StringAttribute | ComplexAttribute | RefAttribute; -export const shortStringFormats = ['id', 'short', 'enum', 'vocabulary']; +export const shortStringFormats = ['id', 'short', 'enum', 'vocabulary', 'version']; export const longStringFormats = ['text', 'json']; // -- GLOBAL -- diff --git a/opencti-platform/opencti-graphql/src/schema/schema-attributes.ts b/opencti-platform/opencti-graphql/src/schema/schema-attributes.ts index 68bec6fa36a1..71bc3e44242e 100644 --- a/opencti-platform/opencti-graphql/src/schema/schema-attributes.ts +++ b/opencti-platform/opencti-graphql/src/schema/schema-attributes.ts @@ -2,7 +2,7 @@ import * as R from 'ramda'; import { GraphQLDateTime } from 'graphql-scalars'; import { RULE_PREFIX } from './general'; import { FunctionalError, UnsupportedError } from '../config/errors'; -import type { AttributeDefinition, AttrType, BasicStoreAttribute, ComplexAttributeWithMappings, MappingDefinition, RawObjectAttribute } from './attribute-definition'; +import type { AttributeDefinition, AttrType, ComplexAttributeWithMappings, MappingDefinition } from './attribute-definition'; import { shortStringFormats } from './attribute-definition'; import { getParentTypes } from './schemaUtils'; import { isFeatureEnabled } from '../config/conf'; @@ -22,11 +22,8 @@ export const depsKeysRegister = { const isMandatoryAttributeDefinition = (schemaDef: AttributeDefinition) => schemaDef.mandatoryType === 'external' || schemaDef.mandatoryType === 'internal'; const isDateAttributeDefinition = (schemaDef: AttributeDefinition) => schemaDef.type === 'date'; -const isNonFlatObjectAttributeDefinition = (schemaDef: AttributeDefinition): schemaDef is ComplexAttributeWithMappings => { // handy typeguard - return schemaDef.type === 'object' && schemaDef.format !== 'flat'; -}; -const isRawObjectAttributeDefinition = (schemaDef: AttributeDefinition): schemaDef is RawObjectAttribute => { // handy typeguard - return schemaDef.type === 'object' && schemaDef.format === 'raw'; +const isObjectAttributeWithMappings = (schemaDef: AttributeDefinition): schemaDef is ComplexAttributeWithMappings => { // handy typeguard + return schemaDef.type === 'object' && (schemaDef.format === 'standard' || schemaDef.format === 'nested'); }; /** @@ -38,7 +35,7 @@ const getAttributeMappingFromPath = (path: string, schemaDef: AttributeDefinitio if (pathTokens.length === 1) { return schemaDef; } - if (!isNonFlatObjectAttributeDefinition(schemaDef)) { + if (!isObjectAttributeWithMappings(schemaDef)) { throw FunctionalError(`Cannot resolve path [${path}], [${schemaDef.name}] is not an object`); } const mapping = schemaDef.mappings.find((m) => m.name === pathTokens[1]); @@ -289,6 +286,10 @@ export const isNumericAttribute = (k: string): boolean => ( export const isDateNumericOrBooleanAttribute = (k: string): boolean => ( schemaAttributesDefinition.isSpecificTypeAttribute(k, 'date', 'numeric', 'boolean') ); +export const isVersionAttribute = (k: string): boolean => { + const attribute = schemaAttributesDefinition.getAttributeByName(k); + return attribute?.type === 'string' && attribute?.format === 'version'; +}; export const isObjectAttribute = (k: string): boolean => ( schemaAttributesDefinition.isSpecificTypeAttribute(k, 'object') ); @@ -327,7 +328,7 @@ const validateInputAgainstSchema = (input: any, schemaDef: AttributeDefinition) } } - if (isNonFlatObjectAttributeDefinition(schemaDef)) { + if (isObjectAttributeWithMappings(schemaDef)) { if (!isMandatory && R.isNil(input)) { return; // nothing to check (happens on 'remove' operation for instance } @@ -339,10 +340,6 @@ const validateInputAgainstSchema = (input: any, schemaDef: AttributeDefinition) throw FunctionalError(`Validation against schema failed on attribute [${schemaDef.name}]: value must be an object`); } - if (isRawObjectAttributeDefinition(schemaDef)) { - return; // raw objects are not recursively validated - } - const inputValues = Array.isArray(input) ? input : [input]; inputValues.forEach((value) => { // check the value adhere to its mapping diff --git a/opencti-platform/opencti-graphql/src/utils/sorting.ts b/opencti-platform/opencti-graphql/src/utils/sorting.ts index 6c5f4c1ea938..824ee68fac72 100644 --- a/opencti-platform/opencti-graphql/src/utils/sorting.ts +++ b/opencti-platform/opencti-graphql/src/utils/sorting.ts @@ -1,9 +1,10 @@ -import { isDateNumericOrBooleanAttribute, schemaAttributesDefinition } from '../schema/schema-attributes'; +import { isDateNumericOrBooleanAttribute, isVersionAttribute, schemaAttributesDefinition } from '../schema/schema-attributes'; import { FunctionalError, UnsupportedError } from '../config/errors'; import { getPirWithAccessCheck } from '../modules/pir/pir-checkPirAccess'; import type { AuthContext, AuthUser } from '../types/user'; const PIR_ORDERING_CRITERIA = ['pir_score', 'last_pir_score_date']; +const VERSION_ORDERING_CRITERIA = ['version', 'support_version']; export const buildElasticSortingForAttributeCriteria = async ( context: AuthContext, @@ -47,10 +48,13 @@ export const buildElasticSortingForAttributeCriteria = async ( // criteria not in schema, attempt keyword sorting as a last resort if (!definition) { + if (VERSION_ORDERING_CRITERIA.includes(orderCriteria)) { + return { [orderCriteria]: { order: orderMode, missing: '_last' } }; + } return { [`${orderCriteria}.keyword`]: { order: orderMode, missing: '_last' } }; } - if (isDateNumericOrBooleanAttribute(orderCriteria)) { + if (isDateNumericOrBooleanAttribute(orderCriteria) || isVersionAttribute(orderCriteria) || VERSION_ORDERING_CRITERIA.includes(orderCriteria)) { // sorting on null dates results to an error, one way to fix it is to use missing: 0 // see https://github.com/elastic/elasticsearch/issues/81960 return { [orderCriteria]: { order: orderMode, missing: definition.type === 'date' ? 0 : '_last' } }; From 4d540513da33f0ac6e7244ec55749f9278e6222b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Tue, 18 Aug 2026 20:58:16 +0200 Subject: [PATCH 06/45] fix(connector): sanitize embedded contracts and runtime config vars --- .../src/domain/connector-migration.ts | 12 +- .../opencti-graphql/src/domain/connector.ts | 4 +- .../src/modules/catalog/catalog-domain.ts | 142 +++++++++++++----- .../src/modules/connector/connector-domain.ts | 3 +- 4 files changed, 116 insertions(+), 45 deletions(-) diff --git a/opencti-platform/opencti-graphql/src/domain/connector-migration.ts b/opencti-platform/opencti-graphql/src/domain/connector-migration.ts index 29998282320f..81144ee9df50 100644 --- a/opencti-platform/opencti-graphql/src/domain/connector-migration.ts +++ b/opencti-platform/opencti-graphql/src/domain/connector-migration.ts @@ -7,7 +7,11 @@ import { completeConnector, connector, connectors } from '../database/repository import type { Connector, ConnectorContractConfiguration, ContractConfigInput } from '../generated/graphql'; import { publishUserAction } from '../listener/UserActionListener'; import { addConnectorDeployedCount } from '../manager/telemetryManager'; -import { computeConnectorTargetContract, mapCatalogContractToGraphqlCatalogContractWithoutExcludedConfigVars } from '../modules/catalog/catalog-domain'; +import { + computeConnectorTargetContract, + mapContractEntityFieldsToEmbeddedConnectorManagerContract, + mapContractEntityFieldsToGraphqlCatalogContract, +} from '../modules/catalog/catalog-domain'; import { ABSTRACT_INTERNAL_OBJECT } from '../schema/general'; import { ENTITY_TYPE_CONNECTOR, ENTITY_TYPE_CONNECTOR_MANAGER } from '../schema/internalObject'; import type { BasicStoreEntityConnectorManager } from '../types/connector'; @@ -148,7 +152,7 @@ export const assessConnectorMigration = async (context: AuthContext, user: AuthU throw FunctionalError('Contract not found', { container_image: containerImage }); } - const contract = mapCatalogContractToGraphqlCatalogContractWithoutExcludedConfigVars(contractData); + const contract = mapContractEntityFieldsToGraphqlCatalogContract(contractData, { excludeRuntimeConfigVars: true }); // Check type are correct if (existingConnector.connector_type !== contract.container_type) { @@ -256,7 +260,7 @@ export const migrateConnectorToManaged = async ( throw FunctionalError('Contract not found', { container_image: containerImage }); } - const contract = mapCatalogContractToGraphqlCatalogContractWithoutExcludedConfigVars(contractData); + const contract = mapContractEntityFieldsToGraphqlCatalogContract(contractData, { excludeRuntimeConfigVars: true }); if (!contract.manager_supported) { throw FunctionalError('Connector is not managed by composer'); @@ -357,7 +361,7 @@ export const migrateConnectorToManaged = async ( title: existingConnector.name, catalog_id: contractData.catalog_id, manager_contract_image: contract.container_image, - manager_contract: contractData, + manager_contract: mapContractEntityFieldsToEmbeddedConnectorManagerContract(contractData), manager_contract_configuration: filteredConfigurations, manager_requested_status: 'stopped', }; diff --git a/opencti-platform/opencti-graphql/src/domain/connector.ts b/opencti-platform/opencti-graphql/src/domain/connector.ts index be363c1117d8..ef515924ae0c 100644 --- a/opencti-platform/opencti-graphql/src/domain/connector.ts +++ b/opencti-platform/opencti-graphql/src/domain/connector.ts @@ -55,7 +55,7 @@ import { isCompatibleVersionWithMinimal } from '../utils/version'; import { extractEntityRepresentativeName } from '../database/entity-representative'; import type { BasicStoreCommon, StoreEntity } from '../types/store'; import { addConnectorDeployedCount, addWorkbenchDraftConvertionCount, addWorkbenchValidationCount } from '../manager/telemetryManager'; -import { computeConnectorTargetContract } from '../modules/catalog/catalog-domain'; +import { computeConnectorTargetContract, mapContractEntityFieldsToEmbeddedConnectorManagerContract } from '../modules/catalog/catalog-domain'; import { getEntitiesMapFromCache } from '../database/cache'; import { createOnTheFlyUser } from '../modules/user/user-domain'; @@ -310,7 +310,7 @@ export const managedConnectorAdd = async ( connector_user_id: connectorUser.id, manager_contract_image: input.manager_contract_image, manager_contract_configuration: contractConfigurations, - manager_contract: targetContract, + manager_contract: mapContractEntityFieldsToEmbeddedConnectorManagerContract(targetContract), manager_requested_status: 'stopped', connector_state_timestamp: now(), built_in: false, diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-domain.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-domain.ts index 09d02638dd34..073f15f84924 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-domain.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-domain.ts @@ -6,20 +6,29 @@ import { type BasicStoreEntityCatalogContract, type BasicStoreEntityCatalogManifest, type CatalogContract, - type CatalogContractDtoV0, type CatalogContractEntityFields, type GraphqlCatalog, type GraphqlCatalogContract, } from './catalog-types'; import { isEmptyField } from '../../database/utils'; import { UnsupportedError } from '../../config/errors'; -import { idGenFromData } from '../../schema/identifier'; -import { logApp } from '../../config/conf'; import type { ConnectorContractConfiguration, ContractConfigInput } from '../../generated/graphql'; import type { ValidateFunction } from 'ajv'; -import { findCatalogContractsByCatalogId, findCatalogManifestByCatalogId, findCatalogs, findLatestCompatibleCatalogContractBySlug } from './catalog-repository'; +import { findCatalogManifestByCatalogId, findCatalogs, findLatestCompatibleCatalogContractBySlug, findLatestCompatibleCatalogContractsByCatalogId } from './catalog-repository'; const validatorCache = new Map(); +const EXCLUDED_CONFIG_VARS = ['OPENCTI_TOKEN', 'OPENCTI_URL', 'CONNECTOR_TYPE', 'CONNECTOR_RUN_AND_TERMINATE']; + +const getContractConfigSchemaWithoutExcludedRuntimeVars = (configSchema: CatalogContract['config_schema']) => { + const filteredProperties = Object.fromEntries( + Object.entries(configSchema.properties).filter(([property]) => !EXCLUDED_CONFIG_VARS.includes(property)), + ); + return { + ...configSchema, + properties: filteredProperties, + required: configSchema.required.filter((property) => !EXCLUDED_CONFIG_VARS.includes(property)), + }; +}; /** * Compiles (or retrieves from cache) an AJV validator for a given schema. @@ -226,7 +235,7 @@ export const validateContractConfigurations = ( contractConfigurations: ConnectorContractConfiguration[], targetContract: Pick, ) => { - const targetConfig = targetContract.config_schema; + const targetConfig = getContractConfigSchemaWithoutExcludedRuntimeVars(targetContract.config_schema); // Build validation object from configurations // For AJV validation, arrays need to be actual arrays, not comma-separated strings @@ -295,7 +304,7 @@ export const computeConnectorTargetContract = ( publicKey: string, currentManagerContractConfiguration?: ConnectorContractConfiguration[], ): ConnectorContractConfiguration[] => { - const targetConfig = targetContract.config_schema; + const targetConfig = getContractConfigSchemaWithoutExcludedRuntimeVars(targetContract.config_schema); // Create maps for efficient lookups const configMap = new Map(configurations.map((c) => [c.key, c])); @@ -347,25 +356,6 @@ export const computeConnectorTargetContract = ( return contractConfigurations; }; -const EXCLUDED_CONFIG_VARS = ['OPENCTI_TOKEN', 'OPENCTI_URL', 'CONNECTOR_TYPE', 'CONNECTOR_RUN_AND_TERMINATE']; - -export const mapCatalogContractToGraphqlCatalogContractWithoutExcludedConfigVars = ( - contract: BasicStoreEntityCatalogContract, -): GraphqlCatalogContract => { - const finalContract = mapContractEntityFieldsToGraphqlCatalogContract(contract); - if (finalContract.manager_supported) { - if (!finalContract.config_schema) { - logApp.warn('A contract has manager_supported=true but is missing config_schema', { contractTitle: finalContract.title }); - } else { - EXCLUDED_CONFIG_VARS.forEach((property) => { - delete finalContract.config_schema.properties[property]; - }); - finalContract.config_schema.required = contract.config_schema.required.filter((item) => !EXCLUDED_CONFIG_VARS.includes(item)); - } - } - return finalContract; -}; - const mapCatalogToGraphqlCatalog = ( catalog: BasicStoreEntityCatalogManifest, contracts: BasicStoreEntityCatalogContract[], @@ -374,11 +364,11 @@ const mapCatalogToGraphqlCatalog = ( id: catalog.catalog_id, name: catalog.name, description: catalog.description, - entity_type: 'Catalog', - parent_types: ['Internal'], - standard_id: idGenFromData('catalog', { id: catalog.id }), + entity_type: catalog.entity_type, + parent_types: catalog.parent_types, + standard_id: catalog.standard_id, contracts: contracts.map((c) => - JSON.stringify(mapCatalogContractToGraphqlCatalogContractWithoutExcludedConfigVars(c)), + JSON.stringify(mapContractEntityFieldsToGraphqlCatalogContract(c, { excludeRuntimeConfigVars: true })), ), }; }; @@ -388,14 +378,17 @@ export const queryCatalogById = async (context: AuthContext, user: AuthUser, cat if (!catalog) { return null; } - const contracts = await findCatalogContractsByCatalogId(context, user, catalogId); - return mapCatalogToGraphqlCatalog(catalog, Object.values(contracts)); + const contracts = await findLatestCompatibleCatalogContractsByCatalogId(context, user, catalogId); + return mapCatalogToGraphqlCatalog(catalog, [...contracts.values()]); }; export const queryCatalogs = async (context: AuthContext, user: AuthUser) => { const catalogs = await findCatalogs(context, user); - const contracts = await Promise.all(catalogs.map((catalog) => findCatalogContractsByCatalogId(context, user, catalog.id))); - return catalogs.map((catalog, idx) => mapCatalogToGraphqlCatalog(catalog, Object.values(contracts[idx]))); + const contracts = await Promise.all(catalogs.map((catalog) => findLatestCompatibleCatalogContractsByCatalogId(context, user, catalog.catalog_id))); + const ret = catalogs.map((catalog, idx) => { + return mapCatalogToGraphqlCatalog(catalog, [...contracts[idx].values()]); + }); + return ret; }; export const queryContractBySlug = async (context: AuthContext, user: AuthUser, contractSlug: string) => { @@ -405,17 +398,23 @@ export const queryContractBySlug = async (context: AuthContext, user: AuthUser, } return { catalog_id: contract.catalog_id, - contract: JSON.stringify(mapCatalogContractToGraphqlCatalogContractWithoutExcludedConfigVars(contract)), + contract: JSON.stringify(mapContractEntityFieldsToGraphqlCatalogContract(contract, { excludeRuntimeConfigVars: true })), }; }; export const mapContractDtoV0ToContractEntityFields = (params: { catalogId: string; - contractDto: CatalogContractDtoV0; + contractDto: CatalogContract; contractContentHash: string; logoUri: string | null; }): CatalogContractEntityFields => { const { catalogId, contractDto, contractContentHash, logoUri } = params; + const supportVersionValue = contractDto.support_version + ? contractDto.support_version.replace(/^\s*>=\s*/, '').trim() + : null; + const normalizedSupportVersion = supportVersionValue && supportVersionValue.length > 0 + ? supportVersionValue + : null; return { catalog_id: catalogId, contract_id: `${contractDto.slug}-${contractDto.container_version}`, @@ -430,11 +429,12 @@ export const mapContractDtoV0ToContractEntityFields = (params: { last_verified_date: contractDto.last_verified_date ?? undefined, playbook_supported: contractDto.playbook_supported, max_confidence_level: contractDto.max_confidence_level, - support_version: contractDto.support_version ?? undefined, + support_version: normalizedSupportVersion ?? undefined, subscription_link: contractDto.subscription_link ?? undefined, source_code: contractDto.source_code ?? undefined, manager_supported: contractDto.manager_supported, version: contractDto.container_version, + contract_version: contractDto.container_version, image: contractDto.container_image, connector_type: contractDto.container_type, config_schema: contractDto.config_schema, @@ -444,7 +444,14 @@ export const mapContractDtoV0ToContractEntityFields = (params: { }; }; -export const mapContractEntityFieldsToGraphqlCatalogContract = (contract: CatalogContractEntityFields): GraphqlCatalogContract => { +export const mapContractEntityFieldsToGraphqlCatalogContract = ( + contract: CatalogContractEntityFields, + options: { excludeRuntimeConfigVars?: boolean } = {}, +): GraphqlCatalogContract => { + const { excludeRuntimeConfigVars = false } = options; + const configSchema = excludeRuntimeConfigVars + ? getContractConfigSchemaWithoutExcludedRuntimeVars(contract.config_schema) + : contract.config_schema; return { title: contract.title, slug: contract.slug, @@ -463,9 +470,68 @@ export const mapContractEntityFieldsToGraphqlCatalogContract = (contract: Catalo container_version: contract.version, container_image: contract.image, container_type: contract.connector_type, - config_schema: contract.config_schema, + config_schema: configSchema, license_type: contract.license_type ?? null, solution_categories: contract.solution_categories ?? [], contact: contract.contact ?? null, }; }; + +export const mapContractEntityFieldsToEmbeddedConnectorManagerContract = ( + contract: CatalogContractEntityFields, +): CatalogContractEntityFields => { + const { + catalog_id, + contract_id, + content_hash, + title, + slug, + description, + short_description, + logo_uri, + use_cases, + verified, + last_verified_date, + playbook_supported, + max_confidence_level, + support_version, + subscription_link, + source_code, + manager_supported, + version, + contract_version, + image, + connector_type, + config_schema, + license_type, + solution_categories, + contact, + } = contract; + return { + catalog_id, + contract_id, + content_hash, + title, + slug, + description, + short_description, + logo_uri, + use_cases, + verified, + last_verified_date, + playbook_supported, + max_confidence_level, + support_version, + subscription_link, + source_code, + manager_supported, + version, + contract_version, + image, + connector_type, + config_schema, + license_type, + solution_categories, + contact, + }; +}; diff --git a/opencti-platform/opencti-graphql/src/modules/connector/connector-domain.ts b/opencti-platform/opencti-graphql/src/modules/connector/connector-domain.ts index f5b0a4fa733d..5d10f0c9f148 100644 --- a/opencti-platform/opencti-graphql/src/modules/connector/connector-domain.ts +++ b/opencti-platform/opencti-graphql/src/modules/connector/connector-domain.ts @@ -3,6 +3,7 @@ import { logApp } from '../../config/conf'; import { publishUserAction } from '../../listener/UserActionListener'; import type { AuthContext, AuthUser } from '../../types/user'; import { findLatestCompatibleCatalogContractBySlug } from '../catalog/catalog-repository'; +import { mapContractEntityFieldsToEmbeddedConnectorManagerContract } from '../catalog/catalog-domain'; import { findManagedConnectorsByCatalogId } from './connector-repository'; import type { BasicStoreEntityConnector } from '../../types/connector'; import { patchAttribute } from '../../database/middleware'; @@ -46,7 +47,7 @@ const autoUpgradeManagedConnector = async ( } // Update connector const patch: Partial = { - manager_contract: latestCompatibleContract, + manager_contract: mapContractEntityFieldsToEmbeddedConnectorManagerContract(latestCompatibleContract), manager_contract_image: latestCompatibleContract.image, }; await patchAttribute(context, user, managedConnector.id, ENTITY_TYPE_CONNECTOR, patch); From d33e389e5eb2f768ea7516e5fa1201d54d3d43e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Tue, 18 Aug 2026 21:37:56 +0200 Subject: [PATCH 07/45] feat(catalog): support custom catalog refresh endpoint source --- .../catalog/sync/catalog-sync-domain.ts | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts b/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts index 842fb644561f..4dfd1777730f 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts @@ -1,4 +1,5 @@ import { createHash } from 'node:crypto'; +import { fileURLToPath, pathToFileURL } from 'node:url'; import conf, { isFeatureEnabled, logApp, PLATFORM_VERSION } from '../../../config/conf'; import { deleteCatalogContracts, @@ -354,6 +355,24 @@ const synchronizeCatalog = async (context: AuthContext, user: AuthUser, sourceCo const EMBEDDED_CATALOG_SYNC_SOURCE: CatalogSyncSourceConfig = { kind: 'embedded', uri: 'embedded' } as const; +const parseCustomCatalogEndpointUri = (uri: string): CatalogSyncSourceConfig => { + if (uri.startsWith('http://') || uri.startsWith('https://')) { + return { kind: 'remote', uri }; + } + if (uri.startsWith('file://')) { + const filepath = fileURLToPath(uri); + return { + kind: 'local', + filepath, + uri, + }; + } + throw UnsupportedError('Unsupported custom catalog endpoint URI protocol', { + uri, + supportedProtocols: ['file://', 'http://', 'https://'], + }); +}; + const initSyncSources = () => { // Build catalog sources configs from env vars/app settings const sources: CatalogSyncSourceConfig[] = []; @@ -363,6 +382,10 @@ const initSyncSources = () => { } else { sources.push(EMBEDDED_CATALOG_SYNC_SOURCE); } + const customCatalogRefreshEndpointUri = conf.get('catalog_manager:custom_catalog_refresh_endpoint_uri'); + if (typeof customCatalogRefreshEndpointUri === 'string' && customCatalogRefreshEndpointUri.length > 0) { + sources.push(parseCustomCatalogEndpointUri(customCatalogRefreshEndpointUri)); + } const CUSTOM_CATALOGS: string[] = conf.get('app:custom_catalogs') ?? []; if (CUSTOM_CATALOGS) { sources.push(...CUSTOM_CATALOGS.map((customCatalog) => { @@ -375,11 +398,11 @@ const initSyncSources = () => { return { kind: 'local' as const, filepath: customCatalog, - uri: 'file:///' + customCatalog, // TODO: normalize + uri: pathToFileURL(customCatalog).toString(), }; })); } - return sources; + return sources.filter((source, idx, all) => all.findIndex((candidate) => candidate.uri === source.uri) === idx); }; const cleanupObsoleteCatalogs = async (context: AuthContext, syncedCatalogs: string[]) => { From 1e55f9ed40ede2c83e36743b7177b9ed12011c31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Tue, 18 Aug 2026 22:04:36 +0200 Subject: [PATCH 08/45] refactor(catalog): remove legacy contract version field --- .../src/modules/catalog/catalog-domain.ts | 5 +--- .../src/modules/catalog/catalog-types.ts | 1 - .../src/modules/catalog/catalog.ts | 3 +-- .../catalog/sync/catalog-sync-domain.ts | 2 -- .../src/modules/connector/connector-domain.ts | 24 +++++++++---------- 5 files changed, 14 insertions(+), 21 deletions(-) diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-domain.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-domain.ts index 073f15f84924..d6fc48f7f769 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-domain.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-domain.ts @@ -433,7 +433,6 @@ export const mapContractDtoV0ToContractEntityFields = (params: { subscription_link: contractDto.subscription_link ?? undefined, source_code: contractDto.source_code ?? undefined, manager_supported: contractDto.manager_supported, - version: contractDto.container_version, contract_version: contractDto.container_version, image: contractDto.container_image, connector_type: contractDto.container_type, @@ -467,7 +466,7 @@ export const mapContractEntityFieldsToGraphqlCatalogContract = ( subscription_link: contract.subscription_link ?? null, source_code: contract.source_code ?? '', manager_supported: contract.manager_supported, - container_version: contract.version, + container_version: contract.contract_version, container_image: contract.image, container_type: contract.connector_type, config_schema: configSchema, @@ -498,7 +497,6 @@ export const mapContractEntityFieldsToEmbeddedConnectorManagerContract = ( subscription_link, source_code, manager_supported, - version, contract_version, image, connector_type, @@ -525,7 +523,6 @@ export const mapContractEntityFieldsToEmbeddedConnectorManagerContract = ( subscription_link, source_code, manager_supported, - version, contract_version, image, connector_type, diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-types.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-types.ts index 6cd8177758d9..d9b01b7d8ff3 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-types.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-types.ts @@ -105,7 +105,6 @@ export interface CatalogContractEntityFields { subscription_link?: string; source_code?: string; manager_supported: boolean; - version: string; contract_version: string; image: string; connector_type: IngestionConnectorType; diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog.ts index 5f38f7ea5c0c..7deea73bf664 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog.ts @@ -29,7 +29,6 @@ export const CATALOG_CONTRACT_MAPPINGS: AttributeDefinition[] = [ { name: 'subscription_link', label: 'Subscription link', type: 'string', format: 'short', mandatoryType: 'no', editDefault: false, multiple: false, upsert: false, isFilterable: false }, { name: 'source_code', label: 'Source code', type: 'string', format: 'short', mandatoryType: 'no', editDefault: false, multiple: false, upsert: false, isFilterable: false }, { name: 'manager_supported', label: 'Manager supported', type: 'boolean', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: true }, - { name: 'version', label: 'Version', type: 'string', format: 'short', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: true }, { name: 'contract_version', label: 'Contract version', type: 'string', format: 'version', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: false }, { name: 'image', label: 'Image', type: 'string', format: 'short', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: true }, { name: 'connector_type', label: 'Connector type', type: 'string', format: 'short', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: true }, @@ -39,7 +38,7 @@ export const CATALOG_CONTRACT_MAPPINGS: AttributeDefinition[] = [ { name: 'contact', label: 'Contact', type: 'string', format: 'short', mandatoryType: 'no', editDefault: false, multiple: false, upsert: false, isFilterable: true }, ]; -// CatalogContract = one persisted entry per (slug, version). +// CatalogContract = one persisted entry per (slug, contract_version). // All connector metadata lives here since any field can vary between versions. const CATALOG_CONTRACT_DEFINITION: ModuleDefinition = { type: { diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts b/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts index 4dfd1777730f..f9c7cd59304c 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts @@ -155,7 +155,6 @@ const computeCatalogSyncOps = (params: { subscription_link: sourceContract.subscription_link ?? undefined, source_code: sourceContract.source_code ?? undefined, manager_supported: sourceContract.manager_supported, - version: sourceContract.container_version, contract_version: sourceContract.container_version, image: sourceContract.container_image, connector_type: sourceContract.container_type, @@ -190,7 +189,6 @@ const computeCatalogSyncOps = (params: { support_version: sourceContract.support_version, subscription_link: sourceContract.subscription_link, manager_supported: sourceContract.manager_supported, - version: sourceContract.container_version, contract_version: sourceContract.container_version, image: sourceContract.container_image, connector_type: sourceContract.container_type, diff --git a/opencti-platform/opencti-graphql/src/modules/connector/connector-domain.ts b/opencti-platform/opencti-graphql/src/modules/connector/connector-domain.ts index 5d10f0c9f148..2c779690f7fd 100644 --- a/opencti-platform/opencti-graphql/src/modules/connector/connector-domain.ts +++ b/opencti-platform/opencti-graphql/src/modules/connector/connector-domain.ts @@ -26,7 +26,7 @@ const autoUpgradeManagedConnector = async ( }); return; } - const { slug, version, content_hash } = manager_contract; + const { slug, contract_version, content_hash } = manager_contract; try { const latestCompatibleContract = await findLatestCompatibleCatalogContractBySlug(context, user, slug); if (!latestCompatibleContract) { @@ -36,12 +36,12 @@ const autoUpgradeManagedConnector = async ( // the connector ? return; } - if (semver.eq(version, latestCompatibleContract.version) + if (semver.eq(contract_version, latestCompatibleContract.contract_version) && content_hash === latestCompatibleContract.content_hash) { logApp.debug('[OPENCTI-MODULE] Managed connector already uses latest compatible version', { module: 'connector', connectorId: managedConnector.id, - version, + version: contract_version, }); return; } @@ -51,13 +51,13 @@ const autoUpgradeManagedConnector = async ( manager_contract_image: latestCompatibleContract.image, }; await patchAttribute(context, user, managedConnector.id, ENTITY_TYPE_CONNECTOR, patch); - if (semver.lt(version, latestCompatibleContract.version)) { + if (semver.lt(contract_version, latestCompatibleContract.contract_version)) { logApp.info('[OPENCTI-MODULE] Upgraded connector to latest compatible version', { module: 'connector', connectorId: managedConnector.id, contractSlug: slug, - previousVersion: version, - newVersion: latestCompatibleContract.version, + previousVersion: contract_version, + newVersion: latestCompatibleContract.contract_version, }); // Activity log // Unsure how correct this is. Maybe the context_data is too big here. @@ -73,13 +73,13 @@ const autoUpgradeManagedConnector = async ( input: patch, }, }); - } else if (semver.gt(version, latestCompatibleContract.version)) { + } else if (semver.gt(contract_version, latestCompatibleContract.contract_version)) { logApp.info('[OPENCTI-MODULE] Downgraded connector to latest compatible version', { module: 'connector', connectorId: managedConnector.id, contractSlug: slug, - previousVersion: version, - newVersion: latestCompatibleContract.version, + previousVersion: contract_version, + newVersion: latestCompatibleContract.contract_version, }); // Activity log // Unsure how correct this is. Maybe the context_data is too big here. @@ -95,12 +95,12 @@ const autoUpgradeManagedConnector = async ( input: patch, }, }); - } else if (semver.eq(version, latestCompatibleContract.version)) { + } else if (semver.eq(contract_version, latestCompatibleContract.contract_version)) { // Shouldn't happen: either a Release issue or a logic/code error. logApp.warn('[OPENCTI-MODULE] Inconsistent connector data, same connector version with different contract content hash', { module: 'connector', contractSlug: slug, - contractVersion: version, + contractVersion: contract_version, }); // Activity log // Unsure how correct this is. Maybe the context_data is too big here. @@ -123,7 +123,7 @@ const autoUpgradeManagedConnector = async ( logApp.error('[OPENCTI-MODULE] Failed to auto-upgrade connector to latest compatible version', { module: 'connector', contractSlug: slug, - contractVersion: version, + contractVersion: contract_version, cause: exception, }); } From 0f333990574aff495234ff6dffca691ecd3042d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Tue, 18 Aug 2026 22:05:14 +0200 Subject: [PATCH 09/45] refactor(catalog): rename catalog manifest entity to catalog --- .../src/modules/catalog/catalog-domain.ts | 8 ++--- .../src/modules/catalog/catalog-repository.ts | 24 ++++++------- .../src/modules/catalog/catalog-types.ts | 10 +++--- .../src/modules/catalog/catalog.ts | 22 ++++++------ .../catalog/sync/catalog-sync-domain.ts | 34 +++++++++---------- 5 files changed, 49 insertions(+), 49 deletions(-) diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-domain.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-domain.ts index d6fc48f7f769..bdab8ecbc882 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-domain.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-domain.ts @@ -4,7 +4,7 @@ import crypto from 'crypto'; import type { AuthContext, AuthUser } from '../../types/user'; import { type BasicStoreEntityCatalogContract, - type BasicStoreEntityCatalogManifest, + type BasicStoreEntityCatalog, type CatalogContract, type CatalogContractEntityFields, type GraphqlCatalog, @@ -14,7 +14,7 @@ import { isEmptyField } from '../../database/utils'; import { UnsupportedError } from '../../config/errors'; import type { ConnectorContractConfiguration, ContractConfigInput } from '../../generated/graphql'; import type { ValidateFunction } from 'ajv'; -import { findCatalogManifestByCatalogId, findCatalogs, findLatestCompatibleCatalogContractBySlug, findLatestCompatibleCatalogContractsByCatalogId } from './catalog-repository'; +import { findCatalogByCatalogId, findCatalogs, findLatestCompatibleCatalogContractBySlug, findLatestCompatibleCatalogContractsByCatalogId } from './catalog-repository'; const validatorCache = new Map(); const EXCLUDED_CONFIG_VARS = ['OPENCTI_TOKEN', 'OPENCTI_URL', 'CONNECTOR_TYPE', 'CONNECTOR_RUN_AND_TERMINATE']; @@ -357,7 +357,7 @@ export const computeConnectorTargetContract = ( }; const mapCatalogToGraphqlCatalog = ( - catalog: BasicStoreEntityCatalogManifest, + catalog: BasicStoreEntityCatalog, contracts: BasicStoreEntityCatalogContract[], ): GraphqlCatalog => { return { @@ -374,7 +374,7 @@ const mapCatalogToGraphqlCatalog = ( }; export const queryCatalogById = async (context: AuthContext, user: AuthUser, catalogId: string) => { - const catalog = await findCatalogManifestByCatalogId(context, user, catalogId); + const catalog = await findCatalogByCatalogId(context, user, catalogId); if (!catalog) { return null; } diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts index 8d0d20ca3ea9..3320131a3495 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts @@ -8,40 +8,40 @@ import type { AuthContext, AuthUser } from '../../types/user'; import { type BasicStoreEntityCatalogContract, ENTITY_TYPE_CATALOG_CONTRACT, - type BasicStoreEntityCatalogManifest, + type BasicStoreEntityCatalog, type CatalogContractCreation, type CatalogContractUpdate, type CatalogContractDeletion, - type CatalogManifestUpsert, - ENTITY_TYPE_CATALOG_MANIFEST, + type CatalogUpsert, + ENTITY_TYPE_CATALOG, } from './catalog-types'; /** * Catalog data accessors & mutators */ -export const findCatalogManifestByCatalogId = async ( +export const findCatalogByCatalogId = async ( context: AuthContext, user: AuthUser, catalogId: string, ) => { - const catalog = await elLoadBy(context, user, 'catalog_id', catalogId, ENTITY_TYPE_CATALOG_MANIFEST); + const catalog = await elLoadBy(context, user, 'catalog_id', catalogId, ENTITY_TYPE_CATALOG); return catalog; }; -export const findCatalogManifestBySourceUri = async ( +export const findCatalogBySourceUri = async ( context: AuthContext, user: AuthUser, sourceUri: string, ) => { - const catalog = await elLoadBy(context, user, 'source_uri', sourceUri); + const catalog = await elLoadBy(context, user, 'source_uri', sourceUri); return catalog; }; -export const upsertCatalog = async (_context: AuthContext, _user: AuthUser, update: CatalogManifestUpsert) => { +export const upsertCatalog = async (_context: AuthContext, _user: AuthUser, update: CatalogUpsert) => { await elIndex(INDEX_INTERNAL_OBJECTS, { ...update, - entity_type: ENTITY_TYPE_CATALOG_MANIFEST, + entity_type: ENTITY_TYPE_CATALOG, }); }; @@ -55,16 +55,16 @@ export const findCatalogs = async (context: AuthContext, user: AuthUser, exclude filterGroups: [], mode: FilterMode.And, } : null; - const catalogs = await fullEntitiesList( + const catalogs = await fullEntitiesList( context, user, - [ENTITY_TYPE_CATALOG_MANIFEST], + [ENTITY_TYPE_CATALOG], { indices: [READ_INDEX_INTERNAL_OBJECTS], filters }, ); return catalogs; }; -export const deleteCatalogs = async (context: AuthContext, catalogEntities: BasicStoreEntityCatalogManifest[]) => { +export const deleteCatalogs = async (context: AuthContext, catalogEntities: BasicStoreEntityCatalog[]) => { await elDeleteInstances(context, catalogEntities); }; diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-types.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-types.ts index d9b01b7d8ff3..2ca9b5d57a99 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-types.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-types.ts @@ -81,7 +81,7 @@ export interface GraphqlCatalog { // region Database types export const ENTITY_TYPE_CATALOG_CONTRACT = 'CatalogContract'; -export const ENTITY_TYPE_CATALOG_MANIFEST = 'CatalogManifest'; +export const ENTITY_TYPE_CATALOG = 'Catalog'; /** * Fields specific to the `CatalogContract` entity. @@ -127,7 +127,7 @@ export interface BasicStoreEntityCatalogContract extends BasicStoreEntity, Catal export interface StoreEntityCatalogContract extends StoreEntity, CatalogContractEntityFields {} -interface CatalogManifestEntityFields { +interface CatalogEntityFields { revision: string; source_uri: string; catalog_id: string; @@ -136,9 +136,9 @@ interface CatalogManifestEntityFields { version: string; } -export interface BasicStoreEntityCatalogManifest extends BasicStoreEntity, CatalogManifestEntityFields {} +export interface BasicStoreEntityCatalog extends BasicStoreEntity, CatalogEntityFields {} -export interface StoreEntityCatalogManifest extends StoreEntity, CatalogManifestEntityFields {} +export interface StoreEntityCatalog extends StoreEntity, CatalogEntityFields {} export interface CatalogContractDeletion { idToDelete: string; @@ -150,5 +150,5 @@ export type CatalogContractUpdate = Required<{ export type CatalogContractCreation = CatalogContractEntityFields & { internal_id: string; standard_id: string }; -export type CatalogManifestUpsert = CatalogManifestEntityFields & { internal_id: string; standard_id: string }; +export type CatalogUpsert = CatalogEntityFields & { internal_id: string; standard_id: string }; // endregion diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog.ts index 7deea73bf664..b6711b021c65 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog.ts @@ -2,11 +2,11 @@ import { type ModuleDefinition, registerDefinition } from '../../schema/module'; import { ABSTRACT_INTERNAL_OBJECT } from '../../schema/general'; import { type BasicStoreEntityCatalogContract, - type BasicStoreEntityCatalogManifest, - ENTITY_TYPE_CATALOG_MANIFEST, + type BasicStoreEntityCatalog, + ENTITY_TYPE_CATALOG, ENTITY_TYPE_CATALOG_CONTRACT, type StoreEntityCatalogContract, - type StoreEntityCatalogManifest, + type StoreEntityCatalog, } from './catalog-types'; import type { AttributeDefinition } from '../../schema/attribute-definition'; import { UnsupportedError } from '../../config/errors'; @@ -67,16 +67,16 @@ const CATALOG_CONTRACT_DEFINITION: ModuleDefinition(CATALOG_CONTRACT_DEFINITION); -const CATALOG_MANIFEST_DEFINITION: ModuleDefinition = { +const CATALOG_DEFINITION: ModuleDefinition = { type: { - id: 'catalogManifests', - name: ENTITY_TYPE_CATALOG_MANIFEST, + id: 'catalogs', + name: ENTITY_TYPE_CATALOG, category: ABSTRACT_INTERNAL_OBJECT, aliased: false, }, identifier: { definition: { - [ENTITY_TYPE_CATALOG_MANIFEST]: [{ src: 'catalog_id' }], + [ENTITY_TYPE_CATALOG]: [{ src: 'catalog_id' }], }, resolvers: {}, }, @@ -90,13 +90,13 @@ const CATALOG_MANIFEST_DEFINITION: ModuleDefinition { - throw UnsupportedError('Catalog manifest representative should not be called'); + throw UnsupportedError('Catalog representative should not be called'); }, converter_2_1: () => { - throw UnsupportedError('Catalog manifest converter_2_1 should not be called'); + throw UnsupportedError('Catalog converter_2_1 should not be called'); }, }; -registerDefinition(CATALOG_MANIFEST_DEFINITION); +registerDefinition(CATALOG_DEFINITION); -export type { BasicStoreEntityCatalogContract, BasicStoreEntityCatalogManifest }; +export type { BasicStoreEntityCatalogContract, BasicStoreEntityCatalog }; diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts b/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts index f9c7cd59304c..d9761c3e6a70 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts @@ -5,8 +5,8 @@ import { deleteCatalogContracts, deleteCatalogs, findCatalogContractsByCatalogId, - findCatalogManifestByCatalogId, - findCatalogManifestBySourceUri, + findCatalogByCatalogId, + findCatalogBySourceUri, findCatalogs, insertCatalogContracts, updateCatalogContracts, @@ -22,16 +22,16 @@ import { } from '../catalog-logo-storage'; import type { BasicStoreEntityCatalogContract, - BasicStoreEntityCatalogManifest, + BasicStoreEntityCatalog, CatalogContractCreation, CatalogContractDeletion, CatalogContractUpdate, - CatalogManifestUpsert, + CatalogUpsert, } from '../catalog-types'; import type { CatalogContractSyncSource, CatalogSyncSource, CatalogSyncSourceConfig } from './catalog-sync-types'; import { fetchSourceCatalog, fetchSourceCatalogRevisionHint } from './catalog-sync-source-gateway'; import { generateStandardId, idGenFromData } from '../../../schema/identifier'; -import { ENTITY_TYPE_CATALOG_CONTRACT, ENTITY_TYPE_CATALOG_MANIFEST } from '../catalog-types'; +import { ENTITY_TYPE_CATALOG_CONTRACT, ENTITY_TYPE_CATALOG } from '../catalog-types'; import { UnsupportedError } from '../../../config/errors'; const DECOUPLING_VERSIONS_FEATURE_FLAG = 'DECOUPLING_VERSIONS'; @@ -57,11 +57,11 @@ const buildCatalogContractIds = (catalogId: string, contractId: string) => { }; }; -const buildCatalogManifestIds = (catalogId: string) => { +const buildCatalogIds = (catalogId: string) => { const keyData = { catalog_id: catalogId }; return { - internal_id: idGenFromData(ENTITY_TYPE_CATALOG_MANIFEST, keyData), - standard_id: generateStandardId(ENTITY_TYPE_CATALOG_MANIFEST, keyData), + internal_id: idGenFromData(ENTITY_TYPE_CATALOG, keyData), + standard_id: generateStandardId(ENTITY_TYPE_CATALOG, keyData), }; }; @@ -123,7 +123,7 @@ const computeCatalogSyncOps = (params: { sourceCatalog: CatalogSyncSource; sourceCatalogContractsHashes: Map; sourceCatalogLogosUris: Map; - currentCatalog: BasicStoreEntityCatalogManifest | undefined; + currentCatalog: BasicStoreEntityCatalog | undefined; currentContracts: Map; }) => { const sourceContractsIds = new Set(params.sourceCatalogContractsHashes.keys()); @@ -207,9 +207,9 @@ const computeCatalogSyncOps = (params: { return; } }); - const catalogManifestIds = buildCatalogManifestIds(params.sourceCatalog.id); - const catalogManifestUpsert: CatalogManifestUpsert = { - ...catalogManifestIds, + const catalogIds = buildCatalogIds(params.sourceCatalog.id); + const catalogUpsert: CatalogUpsert = { + ...catalogIds, revision: params.revision, source_uri: params.sourceConfig.uri, catalog_id: params.sourceCatalog.id, @@ -221,7 +221,7 @@ const computeCatalogSyncOps = (params: { contractsCreations, contractsUpdates, contractsDeletions, - catalogManifestUpsert, + catalogUpsert, }; }; @@ -232,7 +232,7 @@ const synchronizeCatalog = async (context: AuthContext, user: AuthUser, sourceCo sourceUri: sourceConfig.uri, }); const currentCatalogBySourceUri = sourceConfig.kind === 'remote' - ? await findCatalogManifestBySourceUri(context, user, sourceConfig.uri) + ? await findCatalogBySourceUri(context, user, sourceConfig.uri) : undefined; const remoteRevisionHint = sourceConfig.kind === 'remote' ? await fetchSourceCatalogRevisionHint(sourceConfig) @@ -254,7 +254,7 @@ const synchronizeCatalog = async (context: AuthContext, user: AuthUser, sourceCo // Fetch source catalog const sourceCatalog = await fetchSourceCatalog(sourceConfig); // Find existing persisted data in the database corresponding to the catalog id - const currentCatalog = await findCatalogManifestByCatalogId(context, user, sourceCatalog.id); + const currentCatalog = await findCatalogByCatalogId(context, user, sourceCatalog.id); const currentRevision = currentCatalog?.revision; if (currentCatalog) { logApp.debug('[OPENCTI-MODULE] Found existing persisted catalog', { @@ -334,7 +334,7 @@ const synchronizeCatalog = async (context: AuthContext, user: AuthUser, sourceCo await updateCatalogContracts(context, user, catalogSyncDiff.contractsUpdates); } await deleteCatalogContracts(context, user, catalogSyncDiff.contractsDeletions); - await upsertCatalog(context, user, catalogSyncDiff.catalogManifestUpsert); + await upsertCatalog(context, user, catalogSyncDiff.catalogUpsert); const isNotNil = (str: string | null | undefined): str is string => Boolean(str); let usedLogos = catalogSyncDiff.contractsCreations.map(({ logo_uri }) => logo_uri).filter(isNotNil); usedLogos = usedLogos.concat(catalogSyncDiff.contractsUpdates.map(({ logo_uri }) => logo_uri).filter(isNotNil)); @@ -429,7 +429,7 @@ export const synchronizeCatalogs = async (context: AuthContext, user: AuthUser) for (const source of sources) { let result = await synchronizeCatalog(context, user, source); if (source.kind === 'remote' && result.error && decouplingRemoteUri && source.uri === decouplingRemoteUri) { - const alreadyPersistedRemoteCatalog = await findCatalogManifestBySourceUri(context, user, source.uri); + const alreadyPersistedRemoteCatalog = await findCatalogBySourceUri(context, user, source.uri); if (!alreadyPersistedRemoteCatalog) { logApp.error('[OPENCTI-MODULE] Remote catalog source failed before first successful persistence, falling back to embedded source', { sourceKind: source.kind, From 8acfd1f25d81ede2ca3740e29b709274d897d7d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Tue, 18 Aug 2026 22:48:34 +0200 Subject: [PATCH 10/45] chore(catalog): add domain and repository diagnostics logs --- .../src/modules/catalog/catalog-domain.ts | 66 +++++++++++++++++++ .../src/modules/catalog/catalog-repository.ts | 54 ++++++++++++++- 2 files changed, 118 insertions(+), 2 deletions(-) diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-domain.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-domain.ts index bdab8ecbc882..654662ffd8d3 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-domain.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-domain.ts @@ -15,6 +15,7 @@ import { UnsupportedError } from '../../config/errors'; import type { ConnectorContractConfiguration, ContractConfigInput } from '../../generated/graphql'; import type { ValidateFunction } from 'ajv'; import { findCatalogByCatalogId, findCatalogs, findLatestCompatibleCatalogContractBySlug, findLatestCompatibleCatalogContractsByCatalogId } from './catalog-repository'; +import { logApp } from '../../config/conf'; const validatorCache = new Map(); const EXCLUDED_CONFIG_VARS = ['OPENCTI_TOKEN', 'OPENCTI_URL', 'CONNECTOR_TYPE', 'CONNECTOR_RUN_AND_TERMINATE']; @@ -290,9 +291,25 @@ export const validateContractConfigurations = ( ].join('|'); const validate = getOrCompileValidator(cacheKey, jsonValidation); + logApp.debug('[OPENCTI-MODULE] Validating connector contract configuration', { + module: 'catalog', + contractSlug: targetContract.slug, + contractTitle: targetContract.title, + requiredCount: filteredRequired.length, + providedCount: contractConfigurations.length, + validatedFieldsCount: Object.keys(validationProperties).length, + }); const validContractObject = validate(contractObject); if (!validContractObject) { + logApp.warn('[OPENCTI-MODULE] Invalid connector contract configuration', { + module: 'catalog', + contractSlug: targetContract.slug, + contractTitle: targetContract.title, + requiredCount: filteredRequired.length, + providedCount: contractConfigurations.length, + errorsCount: validate.errors?.length ?? 0, + }); const formattedError = formatValidationErrors(validate.errors, targetContract.title); throw UnsupportedError(formattedError, { errors: validate.errors }); } @@ -305,6 +322,9 @@ export const computeConnectorTargetContract = ( currentManagerContractConfiguration?: ConnectorContractConfiguration[], ): ConnectorContractConfiguration[] => { const targetConfig = getContractConfigSchemaWithoutExcludedRuntimeVars(targetContract.config_schema); + let passwordCount = 0; + let defaultedCount = 0; + let reusedCount = 0; // Create maps for efficient lookups const configMap = new Map(configurations.map((c) => [c.key, c])); @@ -318,6 +338,7 @@ export const computeConnectorTargetContract = ( Object.entries(targetConfig.properties).forEach(([propKey, propSchema]) => { const inputConfig = configMap.get(propKey); const existingConfig = currentConfigMap.get(propKey); + const isPassword = propSchema.format === 'password'; // Only process fields that are: // 1. Required (will use default if available) @@ -337,6 +358,16 @@ export const computeConnectorTargetContract = ( return; // Skip this field entirely } + if (!hasInput && !hasExisting && (propSchema.default !== undefined && propSchema.default !== null)) { + defaultedCount += 1; + } + if (hasExisting && !hasInput) { + reusedCount += 1; + } + if (isPassword && hasInput && inputConfig?.value !== existingConfig?.value) { + passwordCount += 1; + } + const finalConfig = resolveConfigurationValue( propKey, propSchema, @@ -352,6 +383,16 @@ export const computeConnectorTargetContract = ( // Validate the configurations validateContractConfigurations(contractConfigurations, targetContract); + logApp.debug('[OPENCTI-MODULE] Computed connector contract configuration', { + module: 'catalog', + contractSlug: targetContract.slug, + contractTitle: targetContract.title, + inputCount: configurations.length, + resolvedCount: contractConfigurations.length, + passwordCount, + defaultedCount, + reusedCount, + }); return contractConfigurations; }; @@ -376,15 +417,30 @@ const mapCatalogToGraphqlCatalog = ( export const queryCatalogById = async (context: AuthContext, user: AuthUser, catalogId: string) => { const catalog = await findCatalogByCatalogId(context, user, catalogId); if (!catalog) { + logApp.debug('[OPENCTI-MODULE] Catalog query by id returned no catalog', { + module: 'catalog', + catalogId, + }); return null; } const contracts = await findLatestCompatibleCatalogContractsByCatalogId(context, user, catalogId); + logApp.debug('[OPENCTI-MODULE] Catalog query by id resolved', { + module: 'catalog', + catalogId, + contractsCount: contracts.size, + }); return mapCatalogToGraphqlCatalog(catalog, [...contracts.values()]); }; export const queryCatalogs = async (context: AuthContext, user: AuthUser) => { const catalogs = await findCatalogs(context, user); const contracts = await Promise.all(catalogs.map((catalog) => findLatestCompatibleCatalogContractsByCatalogId(context, user, catalog.catalog_id))); + const contractsTotalCount = contracts.reduce((total, contractsByCatalog) => total + contractsByCatalog.size, 0); + logApp.debug('[OPENCTI-MODULE] Catalogs query resolved', { + module: 'catalog', + catalogsCount: catalogs.length, + contractsTotalCount, + }); const ret = catalogs.map((catalog, idx) => { return mapCatalogToGraphqlCatalog(catalog, [...contracts[idx].values()]); }); @@ -394,8 +450,18 @@ export const queryCatalogs = async (context: AuthContext, user: AuthUser) => { export const queryContractBySlug = async (context: AuthContext, user: AuthUser, contractSlug: string) => { const contract = await findLatestCompatibleCatalogContractBySlug(context, user, contractSlug); if (!contract) { + logApp.debug('[OPENCTI-MODULE] Contract query by slug returned no contract', { + module: 'catalog', + contractSlug, + }); return null; } + logApp.debug('[OPENCTI-MODULE] Contract query by slug resolved', { + module: 'catalog', + contractSlug, + catalogId: contract.catalog_id, + contractVersion: contract.contract_version, + }); return { catalog_id: contract.catalog_id, contract: JSON.stringify(mapContractEntityFieldsToGraphqlCatalogContract(contract, { excludeRuntimeConfigVars: true })), diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts index 3320131a3495..7e00b7aee5d4 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts @@ -1,4 +1,4 @@ -import { PLATFORM_VERSION } from '../../config/conf'; +import { logApp, PLATFORM_VERSION } from '../../config/conf'; import { elDeleteInstances, elIndex, elIndexElements, elLoadBy } from '../../database/engine'; import { fullEntitiesList } from '../../database/middleware-loader'; import { INDEX_INTERNAL_OBJECTS, READ_INDEX_INTERNAL_OBJECTS } from '../../database/utils'; @@ -147,6 +147,12 @@ export const findLatestCompatibleCatalogContractsByCatalogId = async ( orderMode: OrderingMode.Desc, }, ); + logApp.debug('[OPENCTI-MODULE] Loaded compatible catalog contracts', { + module: 'catalog', + catalogId, + platformVersion: PLATFORM_VERSION, + compatibleContractsCount: contracts.length, + }); // Keep latest compatible version by slug (results are sorted by version desc). return contracts.reduce((map, contract) => { if (map.has(contract.slug)) { @@ -258,7 +264,23 @@ export const findLatestCompatibleCatalogContractByImageName = async ( first: 1, }, ); - return contracts[0]; + const selectedContract = contracts[0]; + if (!selectedContract) { + logApp.debug('[OPENCTI-MODULE] No compatible catalog contract found by image', { + module: 'catalog', + imageName, + platformVersion: PLATFORM_VERSION, + }); + } else { + logApp.debug('[OPENCTI-MODULE] Selected compatible catalog contract by image', { + module: 'catalog', + imageName, + contractId: selectedContract.contract_id, + contractVersion: selectedContract.contract_version, + catalogId: selectedContract.catalog_id, + }); + } + return selectedContract; }; export const insertCatalogContracts = async ( @@ -271,6 +293,13 @@ export const insertCatalogContracts = async ( _index: INDEX_INTERNAL_OBJECTS, entity_type: ENTITY_TYPE_CATALOG_CONTRACT, })); + if (contracts.length > 0) { + logApp.debug('[OPENCTI-MODULE] Inserting catalog contracts', { + module: 'catalog', + count: contracts.length, + catalogIds: [...new Set(contracts.map((contract) => contract.catalog_id))], + }); + } await elIndexElements(context, user, ENTITY_TYPE_CATALOG_CONTRACT, contractsToIndex); }; @@ -285,6 +314,21 @@ export const updateCatalogContracts = async ( _index: INDEX_INTERNAL_OBJECTS, entity_type: ENTITY_TYPE_CATALOG_CONTRACT, })); + if (updates.length > 0) { + if (updates.length > 100) { + logApp.warn('[OPENCTI-MODULE] High volume of catalog contracts updates', { + module: 'catalog', + count: updates.length, + catalogIds: [...new Set(updates.map((update) => update.catalog_id))], + }); + } else { + logApp.debug('[OPENCTI-MODULE] Updating catalog contracts', { + module: 'catalog', + count: updates.length, + catalogIds: [...new Set(updates.map((update) => update.catalog_id))], + }); + } + } await elIndexElements(context, user, ENTITY_TYPE_CATALOG_CONTRACT, contractsToIndex); }; @@ -297,5 +341,11 @@ export const deleteCatalogContracts = async ( _index: INDEX_INTERNAL_OBJECTS, _id: deletion.idToDelete, } as BasicStoreBase)); + if (deletions.length > 0) { + logApp.debug('[OPENCTI-MODULE] Deleting catalog contracts', { + module: 'catalog', + count: deletions.length, + }); + } await elDeleteInstances(context, docs); }; From d80400ec6795ea6a1f69300c5906d6bec0d7a9aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Tue, 18 Aug 2026 22:49:19 +0200 Subject: [PATCH 11/45] refactor(catalog): harden sync source mapping and diagnostics --- .../src/modules/catalog/catalog-resolver.ts | 2 +- .../catalog/sync/catalog-sync-domain.ts | 31 ++++---- .../sync/catalog-sync-source-gateway.ts | 76 ++++++++++++++----- 3 files changed, 76 insertions(+), 33 deletions(-) diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-resolver.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-resolver.ts index e49715ac818e..6b215e578d57 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-resolver.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-resolver.ts @@ -1,5 +1,5 @@ -import { queryCatalogById, queryCatalogs, queryContractBySlug } from './catalog-domain'; import type { Resolvers } from '../../generated/graphql'; +import { queryCatalogById, queryCatalogs, queryContractBySlug } from './catalog-domain'; const catalogResolver: Resolvers = { Query: { diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts b/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts index d9761c3e6a70..c48eb78b9212 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts @@ -1,19 +1,11 @@ import { createHash } from 'node:crypto'; import { fileURLToPath, pathToFileURL } from 'node:url'; import conf, { isFeatureEnabled, logApp, PLATFORM_VERSION } from '../../../config/conf'; -import { - deleteCatalogContracts, - deleteCatalogs, - findCatalogContractsByCatalogId, - findCatalogByCatalogId, - findCatalogBySourceUri, - findCatalogs, - insertCatalogContracts, - updateCatalogContracts, - upsertCatalog, -} from '../catalog-repository'; import { SYSTEM_USER } from '../../../utils/access'; import type { AuthContext, AuthUser } from '../../../types/user'; +import { generateStandardId, idGenFromData } from '../../../schema/identifier'; +import { UnsupportedError } from '../../../config/errors'; +import { ENTITY_TYPE_CATALOG_CONTRACT, ENTITY_TYPE_CATALOG } from '../catalog-types'; import { type CatalogContractLogoUploadOperation, computeCatalogContractLogoUploadOperation, @@ -28,11 +20,19 @@ import type { CatalogContractUpdate, CatalogUpsert, } from '../catalog-types'; +import { + deleteCatalogContracts, + deleteCatalogs, + findCatalogContractsByCatalogId, + findCatalogByCatalogId, + findCatalogBySourceUri, + findCatalogs, + insertCatalogContracts, + updateCatalogContracts, + upsertCatalog, +} from '../catalog-repository'; import type { CatalogContractSyncSource, CatalogSyncSource, CatalogSyncSourceConfig } from './catalog-sync-types'; import { fetchSourceCatalog, fetchSourceCatalogRevisionHint } from './catalog-sync-source-gateway'; -import { generateStandardId, idGenFromData } from '../../../schema/identifier'; -import { ENTITY_TYPE_CATALOG_CONTRACT, ENTITY_TYPE_CATALOG } from '../catalog-types'; -import { UnsupportedError } from '../../../config/errors'; const DECOUPLING_VERSIONS_FEATURE_FLAG = 'DECOUPLING_VERSIONS'; @@ -384,6 +384,9 @@ const initSyncSources = () => { if (typeof customCatalogRefreshEndpointUri === 'string' && customCatalogRefreshEndpointUri.length > 0) { sources.push(parseCustomCatalogEndpointUri(customCatalogRefreshEndpointUri)); } + // We will consider removing the support for these CUSTOM_CATALOGS + // as it historically only supported local files and wasn't explicitly + // documented. const CUSTOM_CATALOGS: string[] = conf.get('app:custom_catalogs') ?? []; if (CUSTOM_CATALOGS) { sources.push(...CUSTOM_CATALOGS.map((customCatalog) => { diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-source-gateway.ts b/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-source-gateway.ts index 3e105cdc7f6a..7597b3081aa5 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-source-gateway.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-source-gateway.ts @@ -1,11 +1,17 @@ import { readFile } from 'node:fs/promises'; -import type { CatalogContract, CatalogDefinition, TypedProperty } from '../catalog-types'; -import type { CatalogContractSyncSource, CatalogSyncSource, CatalogSyncSourceConfig } from './catalog-sync-types'; import conf, { logApp } from '../../../config/conf'; import { isEmptyField } from '../../../database/utils'; import { UnsupportedError } from '../../../config/errors'; import { getOrCompileValidator } from '../catalog-domain'; import { getHttpClient } from '../../../utils/http-client'; +import type { CatalogContract, CatalogDefinition, TypedProperty } from '../catalog-types'; +import type { CatalogContractSyncSource, CatalogSyncSource, CatalogSyncSourceConfig } from './catalog-sync-types'; + +/** + * Data Transfer Objects (DTOs), in several versions. + * - V0: the format used in the embedded catalog manifest, before the decoupling project + * - V1: the format specced during the decoupling project + */ type CatalogContractDtoV0 = CatalogContract; type CatalogDtoV0 = CatalogDefinition; @@ -26,8 +32,8 @@ type CatalogContractDtoV1 = { license_type: 'Free' | 'Commercial' | null; contact: string | null; solution_categories: string[]; - version: string | null; - image_name: string | null; + version: string; + image_name: string; image_type: string; additional_properties: Record; config_schema: Record; @@ -67,7 +73,7 @@ const withAbortTimeout = async (timeoutMs: number, call: (signal: AbortSignal }; class EmbeddedCatalogSyncSource implements CatalogSyncSourceAdapter { - constructor(private _sourceConfig: Extract) {} + constructor(_sourceConfig: Extract) {} async fetch() { const data = await import('../../../__generated__/opencti-manifest.json'); @@ -165,6 +171,10 @@ const DEFAULT_CONFIG_SCHEMA: CatalogContractDtoV0['config_schema'] = { additionalProperties: true, }; +const DEFAULT_PLAYBOOK_SUPPORTED = false; + +const DEFAULT_MAX_CONFIDENCE = 100; + const normalizeConfigSchema = (configSchema: unknown): CatalogContractDtoV0['config_schema'] => { if (!configSchema || typeof configSchema !== 'object' || Array.isArray(configSchema)) { return DEFAULT_CONFIG_SCHEMA; @@ -194,8 +204,11 @@ const normalizeSupportVersion = (supportVersion: string | null | undefined): str return normalized.length > 0 ? normalized : null; }; -const mapCatalogContractDtoV1ToCatalogContractDtoV0 = (contractDto: CatalogContractDtoV1): CatalogContractDtoV0 => { +const mapCatalogContractDtoV1ToCatalogContractSyncSource = (contractDto: CatalogContractDtoV1): CatalogContractSyncSource => { + const playbook_supported = contractDto.additional_properties['playbook_supported']; + const max_confidence_level = contractDto.additional_properties['max_confidence_level']; return { + id: contractDto.id, title: contractDto.title, slug: contractDto.slug, description: contractDto.description, @@ -204,14 +217,20 @@ const mapCatalogContractDtoV1ToCatalogContractDtoV0 = (contractDto: CatalogContr use_cases: contractDto.use_cases, verified: contractDto.verified, last_verified_date: contractDto.last_verified_date, - playbook_supported: false, - max_confidence_level: 100, - support_version: normalizeSupportVersion(contractDto.support_version), + playbook_supported: + typeof playbook_supported === 'boolean' + ? playbook_supported + : DEFAULT_PLAYBOOK_SUPPORTED, + max_confidence_level: + typeof max_confidence_level === 'number' + ? max_confidence_level + : DEFAULT_MAX_CONFIDENCE, + support_version: contractDto.support_version, subscription_link: contractDto.subscription_link, source_code: contractDto.source_code ?? '', manager_supported: contractDto.manager_supported, - container_version: contractDto.version ?? '', - container_image: contractDto.image_name ?? '', + container_version: contractDto.version, + container_image: contractDto.image_name, container_type: contractDto.image_type, config_schema: normalizeConfigSchema(contractDto.config_schema), license_type: contractDto.license_type, @@ -235,7 +254,7 @@ export const mapCatalogContractDtoToCatalogContractSyncSource = ( }; }; -export const mapCatalogDtoToCatalogSyncSource = (catalog: CatalogDtoV0 | CatalogDtoV1) => { +export const mapCatalogDtoToCatalogSyncSource = (catalog: CatalogDtoV0 | CatalogDtoV1): CatalogSyncSource => { if (isCatalogDtoV1(catalog)) { return { id: catalog.id, @@ -243,11 +262,7 @@ export const mapCatalogDtoToCatalogSyncSource = (catalog: CatalogDtoV0 | Catalog description: catalog.description, version: catalog.product_version, contracts: catalog.contracts.map((contractDto) => { - const normalizedContractDto = mapCatalogContractDtoV1ToCatalogContractDtoV0(contractDto); - return { - id: contractDto.id, - ...normalizedContractDto, - }; + return mapCatalogContractDtoV1ToCatalogContractSyncSource(contractDto); }), }; } @@ -263,10 +278,35 @@ export const fetchSourceCatalog = async (sourceConfig: CatalogSyncSourceConfig): const raw = await adapter.fetch(); const syncSource = mapCatalogDtoToCatalogSyncSource(raw); validateSyncSource(syncSource); + logApp.debug('[OPENCTI-MODULE] Fetched and validated catalog source', { + module: 'catalog', + sourceKind: sourceConfig.kind, + sourceUri: sourceConfig.uri, + catalogId: syncSource.id, + contractsCount: syncSource.contracts.length, + manifestSchemaVersion: isCatalogDtoV1(raw) ? raw.manifest_schema_version : '0', + }); return syncSource; }; export const fetchSourceCatalogRevisionHint = async (sourceConfig: CatalogSyncSourceConfig): Promise => { const adapter = getCatalogSourceAdapter(sourceConfig); - return adapter.fetchRevisionHint?.(); + const revisionHint = await adapter.fetchRevisionHint?.(); + if (sourceConfig.kind === 'remote') { + if (revisionHint) { + logApp.debug('[OPENCTI-MODULE] Fetched catalog source revision hint', { + module: 'catalog', + sourceKind: sourceConfig.kind, + sourceUri: sourceConfig.uri, + revisionHint, + }); + } else { + logApp.debug('[OPENCTI-MODULE] Catalog source revision hint unavailable', { + module: 'catalog', + sourceKind: sourceConfig.kind, + sourceUri: sourceConfig.uri, + }); + } + } + return revisionHint; }; From 5b7afbd19e2b10fbf65ae3a8cb1cdcdf4fb9c5b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Wed, 19 Aug 2026 12:02:16 +0200 Subject: [PATCH 12/45] refactor(catalog): align sync model with manifest versioned fields --- .../src/modules/catalog/catalog-types.ts | 4 +- .../src/modules/catalog/catalog.ts | 10 ++- .../catalog/sync/catalog-sync-domain.ts | 4 +- .../sync/catalog-sync-source-gateway.ts | 81 +++++++++++++++---- .../catalog/sync/catalog-sync-types.ts | 4 +- 5 files changed, 79 insertions(+), 24 deletions(-) diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-types.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-types.ts index 2ca9b5d57a99..0c0e01fff38d 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-types.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-types.ts @@ -133,7 +133,9 @@ interface CatalogEntityFields { catalog_id: string; name: string; description: string; - version: string; + product_version: string; + manifest_version: string | null; + manifest_schema_version: '0' | '1'; } export interface BasicStoreEntityCatalog extends BasicStoreEntity, CatalogEntityFields {} diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog.ts index b6711b021c65..23170ccd9f73 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog.ts @@ -81,10 +81,12 @@ const CATALOG_DEFINITION: ModuleDefinition = { resolvers: {}, }, attributes: [ - { name: 'source_uri', label: 'Source URI', type: 'string', format: 'short', mandatoryType: 'external', editDefault: false, multiple: false, upsert: true, isFilterable: true }, - { name: 'catalog_id', label: 'Catalog ID', type: 'string', format: 'short', mandatoryType: 'external', editDefault: false, multiple: false, upsert: true, isFilterable: true }, - { name: 'revision', label: 'Revision', type: 'string', format: 'short', mandatoryType: 'external', editDefault: false, multiple: false, upsert: true, isFilterable: true }, - { name: 'version', label: 'Product version', type: 'string', format: 'short', mandatoryType: 'external', editDefault: false, multiple: false, upsert: true, isFilterable: false }, + { name: 'source_uri', label: 'Source URI', type: 'string', format: 'short', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: true }, + { name: 'catalog_id', label: 'Catalog ID', type: 'string', format: 'short', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: true }, + { name: 'revision', label: 'Revision', type: 'string', format: 'short', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: true }, + { name: 'product_version', label: 'Product version', type: 'string', format: 'short', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: false }, + { name: 'manifest_version', label: 'Manifest version', type: 'string', format: 'short', mandatoryType: 'no', editDefault: false, multiple: false, upsert: false, isFilterable: false }, + { name: 'manifest_schema_version', label: 'Manifest schema version', type: 'string', format: 'short', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: false }, ], relations: [], // Irrelevant as no corresponding Stix object. diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts b/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts index c48eb78b9212..0285ba536b64 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts @@ -215,7 +215,9 @@ const computeCatalogSyncOps = (params: { catalog_id: params.sourceCatalog.id, name: params.sourceCatalog.name, description: params.sourceCatalog.description, - version: params.sourceCatalog.version, + product_version: params.sourceCatalog.product_version, + manifest_version: params.sourceCatalog.manifest_version, + manifest_schema_version: params.sourceCatalog.manifest_schema_version, }; return { contractsCreations, diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-source-gateway.ts b/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-source-gateway.ts index 7597b3081aa5..5f1f6711ad3b 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-source-gateway.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-source-gateway.ts @@ -12,6 +12,7 @@ import type { CatalogContractSyncSource, CatalogSyncSource, CatalogSyncSourceCon * - V0: the format used in the embedded catalog manifest, before the decoupling project * - V1: the format specced during the decoupling project */ +const MANIFEST_SCHEMA_VERSION_1 = '1'; type CatalogContractDtoV0 = CatalogContract; type CatalogDtoV0 = CatalogDefinition; @@ -42,14 +43,18 @@ type CatalogDtoV1 = { id: string; name: string; description: string; - manifest_schema_version: '1'; + manifest_schema_version: typeof MANIFEST_SCHEMA_VERSION_1; manifest_version: string; product_version: string; contracts: Array; }; +interface CatalogDtoWithExplicitSchemaVersion { + manifest_schema_version: string; +} + interface CatalogSyncSourceAdapter { - fetch: () => Promise; + fetch: () => Promise; fetchRevisionHint?: () => Promise; } @@ -239,8 +244,26 @@ const mapCatalogContractDtoV1ToCatalogContractSyncSource = (contractDto: Catalog }; }; -const isCatalogDtoV1 = (catalog: CatalogDtoV0 | CatalogDtoV1): catalog is CatalogDtoV1 => { - return typeof (catalog as CatalogDtoV1).manifest_schema_version === 'string'; +const isCatalogDtoV1 = (catalog: CatalogDtoWithExplicitSchemaVersion): catalog is CatalogDtoV1 => { + return catalog.manifest_schema_version === MANIFEST_SCHEMA_VERSION_1; +}; + +const isCatalogDtoV0 = (catalog: unknown): catalog is CatalogDtoV0 => { + return typeof catalog === 'object' + && catalog !== null + && 'id' in catalog + && 'version' in catalog + && typeof catalog.id === 'string' + && typeof catalog.version === 'string'; +}; + +const isCatalogDtoWithExplicitSchemaVersion = ( + catalog: unknown, +): catalog is CatalogDtoWithExplicitSchemaVersion => { + return typeof catalog === 'object' + && catalog !== null + && 'manifest_schema_version' in catalog + && typeof catalog.manifest_schema_version === 'string'; }; export const mapCatalogContractDtoToCatalogContractSyncSource = ( @@ -254,23 +277,45 @@ export const mapCatalogContractDtoToCatalogContractSyncSource = ( }; }; -export const mapCatalogDtoToCatalogSyncSource = (catalog: CatalogDtoV0 | CatalogDtoV1): CatalogSyncSource => { +const mapCatalogDtoToCatalogSyncSource = (catalog: unknown): CatalogSyncSource => { + if (isCatalogDtoV0(catalog)) { + return mapCatalogDtoV0ToCatalogSyncSource(catalog); + } + if (!isCatalogDtoWithExplicitSchemaVersion(catalog)) { + throw UnsupportedError('Unrecognized catalog format: no manifest_schema_version'); + } if (isCatalogDtoV1(catalog)) { - return { - id: catalog.id, - name: catalog.name, - description: catalog.description, - version: catalog.product_version, - contracts: catalog.contracts.map((contractDto) => { - return mapCatalogContractDtoV1ToCatalogContractSyncSource(contractDto); - }), - }; + return mapCatalogDtoV1ToCatalogSyncSource(catalog); } - const syncSource = { + throw UnsupportedError('Unsupported catalog schema version', { + cause: { + manifest_schema_version: catalog.manifest_schema_version, + }, + }); +}; + +const mapCatalogDtoV1ToCatalogSyncSource = (catalog: CatalogDtoV1): CatalogSyncSource => { + return { + id: catalog.id, + name: catalog.name, + description: catalog.description, + product_version: catalog.product_version, + manifest_version: catalog.manifest_version, + manifest_schema_version: catalog.manifest_schema_version, + contracts: catalog.contracts.map((contractDto) => { + return mapCatalogContractDtoV1ToCatalogContractSyncSource(contractDto); + }), + }; +}; + +const mapCatalogDtoV0ToCatalogSyncSource = (catalog: CatalogDtoV0): CatalogSyncSource => { + return { ...catalog, + product_version: catalog.version, + manifest_version: null, + manifest_schema_version: '0' as const, contracts: catalog.contracts.map(mapCatalogContractDtoToCatalogContractSyncSource), }; - return syncSource; }; export const fetchSourceCatalog = async (sourceConfig: CatalogSyncSourceConfig): Promise => { @@ -284,7 +329,9 @@ export const fetchSourceCatalog = async (sourceConfig: CatalogSyncSourceConfig): sourceUri: sourceConfig.uri, catalogId: syncSource.id, contractsCount: syncSource.contracts.length, - manifestSchemaVersion: isCatalogDtoV1(raw) ? raw.manifest_schema_version : '0', + manifestSchemaVersion: syncSource.manifest_schema_version, + productVersion: syncSource.product_version, + manifestVersion: syncSource.manifest_version, }); return syncSource; }; diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-types.ts b/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-types.ts index a5e91c8dece2..ac6c97013fbc 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-types.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-types.ts @@ -20,6 +20,8 @@ export interface CatalogSyncSource { id: string; name: string; description: string; - version: string; + product_version: string; + manifest_version: string | null; + manifest_schema_version: '0' | '1'; contracts: Array; } From db7a4d9486fb905d757bf70395c488554198f121 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Wed, 19 Aug 2026 12:02:18 +0200 Subject: [PATCH 13/45] fix(catalog): keep logo-view context available in error logs --- .../opencti-graphql/src/modules/catalog/catalog-http.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-http.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-http.ts index d3a6b5ca2154..0e188ced3820 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-http.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-http.ts @@ -8,9 +8,11 @@ import { ResourceNotFoundError, UnsupportedError, UnknownError } from '../../con export const CATALOG_LOGO_VIEW_PATH = '/catalog/logo/*file'; +const CONTEXT_SOURCE = 'catalog_logo_view'; + export const handleCatalogLogoViewRequest: RequestHandler = async (req, res) => { try { - const context = await createAuthenticatedContext(req, res, 'catalog_logo_view'); + const context = await createAuthenticatedContext(req, res, CONTEXT_SOURCE); if (!context.user) { res.sendStatus(403); return; From 16012f881d10d0f769294c50ee3745b5a508417a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Wed, 19 Aug 2026 14:27:32 +0200 Subject: [PATCH 14/45] fix(catalog): harden sync source selection and contract mapping --- .../src/database/repository.js | 6 ++-- .../src/modules/catalog/catalog-domain.ts | 32 +++++++++++++++---- .../src/modules/catalog/catalog-repository.ts | 6 ++-- .../src/modules/catalog/catalog.ts | 2 +- .../catalog/sync/catalog-sync-domain.ts | 31 +++++++++++------- 5 files changed, 52 insertions(+), 25 deletions(-) diff --git a/opencti-platform/opencti-graphql/src/database/repository.js b/opencti-platform/opencti-graphql/src/database/repository.js index f6bcdc0a1d19..f52a749fe057 100644 --- a/opencti-platform/opencti-graphql/src/database/repository.js +++ b/opencti-platform/opencti-graphql/src/database/repository.js @@ -169,14 +169,14 @@ export const computeManagerConnectorImage = async (cn) => { return ''; } const managerContract = cn.manager_contract; - if (!managerContract.image || !managerContract.version) { + if (!managerContract.image || !managerContract.contract_version) { throw FunctionalError('Invalid manager contract snapshot', { connectorId: cn.id ?? cn.internal_id, image: managerContract.image, - version: managerContract.version, + version: managerContract.contract_version, }); } - return `${managerContract.image}:${managerContract.version}`; + return `${managerContract.image}:${managerContract.contract_version}`; }; export const computeManagerContractHash = async (context, user, cn) => { diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-domain.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-domain.ts index 654662ffd8d3..5698cddf09cd 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-domain.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-domain.ts @@ -20,14 +20,33 @@ import { logApp } from '../../config/conf'; const validatorCache = new Map(); const EXCLUDED_CONFIG_VARS = ['OPENCTI_TOKEN', 'OPENCTI_URL', 'CONNECTOR_TYPE', 'CONNECTOR_RUN_AND_TERMINATE']; -const getContractConfigSchemaWithoutExcludedRuntimeVars = (configSchema: CatalogContract['config_schema']) => { +const normalizeContractConfigSchema = ( + configSchema: CatalogContract['config_schema'] | null | undefined, +): CatalogContract['config_schema'] => { + const schema = (configSchema && typeof configSchema === 'object') ? configSchema : {} as CatalogContract['config_schema']; + const properties = (schema.properties && typeof schema.properties === 'object' && !Array.isArray(schema.properties)) + ? schema.properties + : {}; + const required = Array.isArray(schema.required) ? schema.required.filter((property) => typeof property === 'string') : []; + return { + $schema: typeof schema.$schema === 'string' ? schema.$schema : 'https://json-schema.org/draft/2020-12/schema', + $id: typeof schema.$id === 'string' ? schema.$id : '', + type: typeof schema.type === 'string' ? schema.type : 'object', + properties, + required, + additionalProperties: typeof schema.additionalProperties === 'boolean' ? schema.additionalProperties : true, + }; +}; + +const getContractConfigSchemaWithoutExcludedRuntimeVars = (configSchema: CatalogContract['config_schema'] | null | undefined) => { + const normalizedConfigSchema = normalizeContractConfigSchema(configSchema); const filteredProperties = Object.fromEntries( - Object.entries(configSchema.properties).filter(([property]) => !EXCLUDED_CONFIG_VARS.includes(property)), + Object.entries(normalizedConfigSchema.properties).filter(([property]) => !EXCLUDED_CONFIG_VARS.includes(property)), ); return { - ...configSchema, + ...normalizedConfigSchema, properties: filteredProperties, - required: configSchema.required.filter((property) => !EXCLUDED_CONFIG_VARS.includes(property)), + required: normalizedConfigSchema.required.filter((property) => !EXCLUDED_CONFIG_VARS.includes(property)), }; }; @@ -514,9 +533,10 @@ export const mapContractEntityFieldsToGraphqlCatalogContract = ( options: { excludeRuntimeConfigVars?: boolean } = {}, ): GraphqlCatalogContract => { const { excludeRuntimeConfigVars = false } = options; + const normalizedConfigSchema = normalizeContractConfigSchema(contract.config_schema); const configSchema = excludeRuntimeConfigVars - ? getContractConfigSchemaWithoutExcludedRuntimeVars(contract.config_schema) - : contract.config_schema; + ? getContractConfigSchemaWithoutExcludedRuntimeVars(normalizedConfigSchema) + : normalizedConfigSchema; return { title: contract.title, slug: contract.slug, diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts index 7e00b7aee5d4..ae301cc3aaf1 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts @@ -47,11 +47,11 @@ export const upsertCatalog = async (_context: AuthContext, _user: AuthUser, upda export const findCatalogs = async (context: AuthContext, user: AuthUser, excludedIds?: string[]) => { const filters = excludedIds?.length ? { - filters: [{ + filters: excludedIds.map((catalogId) => ({ key: ['catalog_id'], - values: excludedIds, + values: [catalogId], operator: FilterOperator.NotEq, - }], + })), filterGroups: [], mode: FilterMode.And, } : null; diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog.ts index 23170ccd9f73..ae1dce8a20b8 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog.ts @@ -84,7 +84,7 @@ const CATALOG_DEFINITION: ModuleDefinition = { { name: 'source_uri', label: 'Source URI', type: 'string', format: 'short', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: true }, { name: 'catalog_id', label: 'Catalog ID', type: 'string', format: 'short', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: true }, { name: 'revision', label: 'Revision', type: 'string', format: 'short', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: true }, - { name: 'product_version', label: 'Product version', type: 'string', format: 'short', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: false }, + { name: 'product_version', label: 'Catalog product version', type: 'string', format: 'short', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: false }, { name: 'manifest_version', label: 'Manifest version', type: 'string', format: 'short', mandatoryType: 'no', editDefault: false, multiple: false, upsert: false, isFilterable: false }, { name: 'manifest_schema_version', label: 'Manifest schema version', type: 'string', format: 'short', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: false }, ], diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts b/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts index 0285ba536b64..1b7d5fde3c44 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts @@ -36,7 +36,7 @@ import { fetchSourceCatalog, fetchSourceCatalogRevisionHint } from './catalog-sy const DECOUPLING_VERSIONS_FEATURE_FLAG = 'DECOUPLING_VERSIONS'; -const getDecouplingRemoteUri = (): string | undefined => { +const getFiligranCatalogRemoteUri = (): string | undefined => { const xtmHubUrl = conf.get('xtm:xtmhub_url'); if (!isFeatureEnabled(DECOUPLING_VERSIONS_FEATURE_FLAG) || typeof xtmHubUrl !== 'string' || xtmHubUrl.length === 0) { return undefined; @@ -376,16 +376,22 @@ const parseCustomCatalogEndpointUri = (uri: string): CatalogSyncSourceConfig => const initSyncSources = () => { // Build catalog sources configs from env vars/app settings const sources: CatalogSyncSourceConfig[] = []; - const decouplingRemoteUri = getDecouplingRemoteUri(); - if (decouplingRemoteUri) { - sources.push({ kind: 'remote', uri: decouplingRemoteUri }); + if (isFeatureEnabled(DECOUPLING_VERSIONS_FEATURE_FLAG)) { + const customCatalogRefreshEndpointUri = conf.get('catalog_manager:custom_catalog_refresh_endpoint_uri'); + if (typeof customCatalogRefreshEndpointUri === 'string' && customCatalogRefreshEndpointUri.length > 0) { + // Custom decoupling source overrides XTM hub source when configured. + sources.push(parseCustomCatalogEndpointUri(customCatalogRefreshEndpointUri)); + } else { + const decouplingRemoteUri = getFiligranCatalogRemoteUri(); + if (decouplingRemoteUri) { + sources.push({ kind: 'remote', uri: decouplingRemoteUri }); + } else { + sources.push(EMBEDDED_CATALOG_SYNC_SOURCE); + } + } } else { sources.push(EMBEDDED_CATALOG_SYNC_SOURCE); } - const customCatalogRefreshEndpointUri = conf.get('catalog_manager:custom_catalog_refresh_endpoint_uri'); - if (typeof customCatalogRefreshEndpointUri === 'string' && customCatalogRefreshEndpointUri.length > 0) { - sources.push(parseCustomCatalogEndpointUri(customCatalogRefreshEndpointUri)); - } // We will consider removing the support for these CUSTOM_CATALOGS // as it historically only supported local files and wasn't explicitly // documented. @@ -422,18 +428,19 @@ const cleanupObsoleteCatalogs = async (context: AuthContext, syncedCatalogs: str export const synchronizeCatalogs = async (context: AuthContext, user: AuthUser) => { const sources = initSyncSources(); - const decouplingRemoteUri = getDecouplingRemoteUri(); + const filigranCatalogRemoteUri = getFiligranCatalogRemoteUri(); logApp.debug('[OPENCTI-MODULE] Synchronizing catalogs', { - count: sources.length, - decouplingRemoteUri, module: 'catalog', + executionContext: context.source, + count: sources.length, + filigranCatalogRemoteUri, }); const syncedCatalogs: string[] = []; const syncedCatalogsWithChanges: string[] = []; // Sync catalogs from sources for (const source of sources) { let result = await synchronizeCatalog(context, user, source); - if (source.kind === 'remote' && result.error && decouplingRemoteUri && source.uri === decouplingRemoteUri) { + if (source.kind === 'remote' && result.error && filigranCatalogRemoteUri && source.uri === filigranCatalogRemoteUri) { const alreadyPersistedRemoteCatalog = await findCatalogBySourceUri(context, user, source.uri); if (!alreadyPersistedRemoteCatalog) { logApp.error('[OPENCTI-MODULE] Remote catalog source failed before first successful persistence, falling back to embedded source', { From 513a817a3eba3465a45682be6318c87ba330744f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Wed, 19 Aug 2026 14:27:46 +0200 Subject: [PATCH 15/45] test(catalog): expand unit and integration coverage --- .../src/utils/filters/filtersUtils.tsx | 4 +- .../migration-embed-catalog-contract-test.ts | 216 ++++++++++++++++ .../repository-manager-contract-test.ts | 83 +++++++ .../catalog-domain/contract-mapping-test.ts | 138 +++++++++++ .../modules/catalog/catalog-http-test.ts | 106 ++++++++ .../catalog/catalog-logo-storage-test.ts | 128 ++++++++++ .../catalog/catalog-sync-domain-test.ts | 232 ++++++++++++++++++ .../catalog-sync-source-gateway-test.ts | 228 +++++++++++++++++ .../connector-domain-auto-upgrade-test.ts | 189 ++++++++++++++ .../tests/01-unit/utils/data-url-test.ts | 48 ++++ .../tests/01-unit/utils/logger-test.ts | 11 +- .../tests/01-unit/utils/sorting-test.ts | 8 + .../02-resolvers/connector-composer-test.ts | 14 ++ .../02-resolvers/connector-migration-test.ts | 4 + .../catalog/catalog-resolver-test.ts | 199 +++++++++++++++ .../fixture/integration-catalog-v0.json | 50 ++++ .../fixture/integration-catalog-v1.json | 54 ++++ .../10-modules/http/catalog-logo-http-test.ts | 33 +++ 18 files changed, 1742 insertions(+), 3 deletions(-) create mode 100644 opencti-platform/opencti-graphql/tests/01-unit/database/migration-embed-catalog-contract-test.ts create mode 100644 opencti-platform/opencti-graphql/tests/01-unit/database/repository-manager-contract-test.ts create mode 100644 opencti-platform/opencti-graphql/tests/01-unit/domain/catalog-domain/contract-mapping-test.ts create mode 100644 opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-http-test.ts create mode 100644 opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-logo-storage-test.ts create mode 100644 opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-sync-domain-test.ts create mode 100644 opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-sync-source-gateway-test.ts create mode 100644 opencti-platform/opencti-graphql/tests/01-unit/modules/connector/connector-domain-auto-upgrade-test.ts create mode 100644 opencti-platform/opencti-graphql/tests/01-unit/utils/data-url-test.ts create mode 100644 opencti-platform/opencti-graphql/tests/03-integration/10-modules/catalog/catalog-resolver-test.ts create mode 100644 opencti-platform/opencti-graphql/tests/03-integration/10-modules/catalog/fixture/integration-catalog-v0.json create mode 100644 opencti-platform/opencti-graphql/tests/03-integration/10-modules/catalog/fixture/integration-catalog-v1.json create mode 100644 opencti-platform/opencti-graphql/tests/03-integration/10-modules/http/catalog-logo-http-test.ts diff --git a/opencti-platform/opencti-front/src/utils/filters/filtersUtils.tsx b/opencti-platform/opencti-front/src/utils/filters/filtersUtils.tsx index 33401d01d57d..4d5d31288f6a 100644 --- a/opencti-platform/opencti-front/src/utils/filters/filtersUtils.tsx +++ b/opencti-platform/opencti-front/src/utils/filters/filtersUtils.tsx @@ -222,7 +222,7 @@ export const isFilterGroupFormatCorrect = (value: unknown): boolean => { export const isUniqFilter = (key: string, filterKeysSchema: Map>) => { const filterDefinition = filterKeysSchema.get('Stix-Core-Object')?.get(key); - return !!(filterDefinition && ['boolean', 'date', 'integer', 'float', 'version'].includes(filterDefinition.type)); + return !!(filterDefinition && ['boolean', 'date', 'integer', 'float'].includes(filterDefinition.type)); }; // basic text filters are filters of type string or text that are not entity types filters @@ -238,7 +238,7 @@ export const isBasicTextFilter = ( export const isNumericFilter = ( filterType?: string, ) => { - return filterType === 'integer' || filterType === 'float' || filterType === 'version'; + return filterType === 'integer' || filterType === 'float'; }; /** diff --git a/opencti-platform/opencti-graphql/tests/01-unit/database/migration-embed-catalog-contract-test.ts b/opencti-platform/opencti-graphql/tests/01-unit/database/migration-embed-catalog-contract-test.ts new file mode 100644 index 000000000000..69df7c4cb70d --- /dev/null +++ b/opencti-platform/opencti-graphql/tests/01-unit/database/migration-embed-catalog-contract-test.ts @@ -0,0 +1,216 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const { + mockFullEntitiesList, + mockPatchAttribute, + mockListCatalogContractLogos, + mockStoreCatalogContractLogo, + mockComputeContractContentHash, + mockMapCatalogContractDtoToCatalogContractSyncSource, + mockMapContractDtoV0ToContractEntityFields, +} = vi.hoisted(() => ({ + mockFullEntitiesList: vi.fn(), + mockPatchAttribute: vi.fn(), + mockListCatalogContractLogos: vi.fn(), + mockStoreCatalogContractLogo: vi.fn(), + mockComputeContractContentHash: vi.fn(), + mockMapCatalogContractDtoToCatalogContractSyncSource: vi.fn(), + mockMapContractDtoV0ToContractEntityFields: vi.fn(), +})); + +vi.mock('../../../src/config/conf', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + default: { + ...actual.default, + get: vi.fn((key: string) => { + if (key === 'redis:ca') return []; + if (key === 'redis:use_ssl') return false; + return undefined; + }), + }, + logMigration: { + info: vi.fn(), + error: vi.fn(), + warn: vi.fn(), + debug: vi.fn(), + }, + }; +}); + +vi.mock('../../../src/database/middleware-loader', () => ({ + fullEntitiesList: mockFullEntitiesList, +})); + +vi.mock('../../../src/database/middleware', () => ({ + patchAttribute: mockPatchAttribute, +})); + +vi.mock('../../../src/utils/access', () => ({ + executionContext: vi.fn(() => ({ source: 'migration' })), + SYSTEM_USER: { id: 'system-user' }, +})); + +vi.mock('../../../src/modules/catalog/catalog-logo-storage', () => ({ + listCatalogContractLogos: mockListCatalogContractLogos, + storeCatalogContractLogo: mockStoreCatalogContractLogo, +})); + +vi.mock('../../../src/modules/catalog/sync/catalog-sync-domain', () => ({ + computeContractContentHash: mockComputeContractContentHash, +})); + +vi.mock('../../../src/modules/catalog/sync/catalog-sync-source-gateway', () => ({ + mapCatalogContractDtoToCatalogContractSyncSource: mockMapCatalogContractDtoToCatalogContractSyncSource, +})); + +vi.mock('../../../src/modules/catalog/catalog-domain', () => ({ + mapContractDtoV0ToContractEntityFields: mockMapContractDtoV0ToContractEntityFields, +})); + +vi.mock('../../../src/__generated__/opencti-manifest.json', () => ({ + default: { + id: 'filigran-catalog', + contracts: [ + { + title: 'IPinfo', + slug: 'ipinfo', + description: 'desc', + short_description: 'short', + logo: null, + use_cases: [], + verified: true, + last_verified_date: '2024-01-01', + playbook_supported: false, + max_confidence_level: 50, + support_version: '6.7.0', + subscription_link: null, + source_code: '', + manager_supported: true, + container_version: '1.2.3', + container_image: 'opencti/connector-ipinfo', + container_type: 'EXTERNAL_IMPORT', + config_schema: { + $schema: 'https://json-schema.org/draft/2020-12/schema', + $id: 'schema-id', + type: 'object', + properties: {}, + required: [], + additionalProperties: true, + }, + license_type: null, + solution_categories: [], + contact: null, + }, + ], + }, +})); + +import { up } from '../../../src/migrations/1786264797376-embed-catalog-contract-in-connector-entity'; + +const runUp = async () => { + await new Promise((resolve, reject) => { + up((error?: Error) => { + if (error) reject(error); + else resolve(); + }); + }); +}; + +describe('migration 1786264797376 embed catalog contract', () => { + beforeEach(() => { + vi.clearAllMocks(); + mockListCatalogContractLogos.mockResolvedValue(new Set()); + mockStoreCatalogContractLogo.mockResolvedValue({ result: 'success', existed: false, logoUri: '/storage/view/catalog-logos/logo.png', filename: 'logo.png' }); + mockMapCatalogContractDtoToCatalogContractSyncSource.mockReturnValue({ id: 'ipinfo-1.2.3' }); + mockComputeContractContentHash.mockReturnValue('hash-1'); + mockMapContractDtoV0ToContractEntityFields.mockReturnValue({ contract_id: 'ipinfo-1.2.3', image: 'opencti/connector-ipinfo' }); + mockPatchAttribute.mockResolvedValue(undefined); + }); + + it('should patch missing manager_contract and missing manager_upgrade_strategy', async () => { + mockFullEntitiesList.mockResolvedValue([{ + id: 'connector-1', + catalog_id: 'filigran-catalog', + manager_contract_image: 'opencti/connector-ipinfo', + manager_contract: undefined, + manager_upgrade_strategy: undefined, + }]); + await runUp(); + expect(mockPatchAttribute).toHaveBeenCalledWith( + { source: 'migration' }, + { id: 'system-user' }, + 'connector-1', + 'Connector', + expect.objectContaining({ + manager_upgrade_strategy: 'latest', + }), + ); + const patch = mockPatchAttribute.mock.calls[0][4]; + expect(patch.manager_contract).toBeDefined(); + }); + + it('should only patch upgrade strategy when manager contract snapshot already exists', async () => { + mockFullEntitiesList.mockResolvedValue([{ + id: 'connector-2', + catalog_id: 'filigran-catalog', + manager_contract_image: 'opencti/connector-ipinfo', + manager_contract: { contract_id: 'ipinfo-1.2.3' }, + manager_upgrade_strategy: undefined, + }]); + await runUp(); + const patch = mockPatchAttribute.mock.calls[0][4]; + expect(patch.manager_upgrade_strategy).toBe('latest'); + expect(patch.manager_contract).toBeUndefined(); + }); + + it('should not patch connectors that already have manager contract and strategy', async () => { + mockFullEntitiesList.mockResolvedValue([{ + id: 'connector-3', + catalog_id: 'filigran-catalog', + manager_contract_image: 'opencti/connector-ipinfo', + manager_contract: { contract_id: 'ipinfo-1.2.3' }, + manager_upgrade_strategy: 'latest', + }]); + await runUp(); + expect(mockPatchAttribute).not.toHaveBeenCalled(); + }); + + it('should still patch upgrade strategy when contract is missing from manifest', async () => { + mockFullEntitiesList.mockResolvedValue([{ + id: 'connector-4', + catalog_id: 'filigran-catalog', + manager_contract_image: 'opencti/connector-unknown', + manager_contract: undefined, + manager_upgrade_strategy: undefined, + }]); + await runUp(); + expect(mockPatchAttribute).toHaveBeenCalledWith( + { source: 'migration' }, + { id: 'system-user' }, + 'connector-4', + 'Connector', + { manager_upgrade_strategy: 'latest' }, + ); + }); + + it('should patch manager_contract even when logo storage fails', async () => { + mockStoreCatalogContractLogo.mockResolvedValue({ + result: 'failed', + logoUri: null, + error: new Error('logo upload failed'), + }); + mockFullEntitiesList.mockResolvedValue([{ + id: 'connector-5', + catalog_id: 'filigran-catalog', + manager_contract_image: 'opencti/connector-ipinfo', + manager_contract: undefined, + manager_upgrade_strategy: undefined, + }]); + await runUp(); + const patch = mockPatchAttribute.mock.calls[0][4]; + expect(patch.manager_contract).toBeDefined(); + expect(patch.manager_upgrade_strategy).toBe('latest'); + }); +}); diff --git a/opencti-platform/opencti-graphql/tests/01-unit/database/repository-manager-contract-test.ts b/opencti-platform/opencti-graphql/tests/01-unit/database/repository-manager-contract-test.ts new file mode 100644 index 000000000000..7feb9166b1d3 --- /dev/null +++ b/opencti-platform/opencti-graphql/tests/01-unit/database/repository-manager-contract-test.ts @@ -0,0 +1,83 @@ +import { describe, expect, it } from 'vitest'; +import { FunctionalError } from '../../../src/config/errors'; +import { + computeManagerConnectorContract, + computeManagerConnectorExcerpt, + computeManagerConnectorImage, +} from '../../../src/database/repository'; + +const connectorWithContract = () => ({ + id: 'connector-1', + internal_id: 'connector-1', + manager_contract: { + catalog_id: 'catalog-1', + contract_id: 'ipinfo-1.2.3', + content_hash: 'hash-1', + title: 'IPinfo', + slug: 'ipinfo', + description: 'desc', + short_description: 'short', + logo_uri: '/storage/view/catalog-logos/logo.png', + use_cases: [], + verified: true, + last_verified_date: '2024-01-01', + playbook_supported: false, + max_confidence_level: 50, + support_version: '6.7.0', + subscription_link: null, + source_code: '', + manager_supported: true, + contract_version: '1.2.3', + image: 'opencti/connector-ipinfo', + connector_type: 'EXTERNAL_IMPORT', + config_schema: { + $schema: 'https://json-schema.org/draft/2020-12/schema', + $id: 'schema-id', + type: 'object', + properties: {}, + required: [], + additionalProperties: true, + }, + license_type: null, + solution_categories: [], + contact: null, + }, +}); + +describe('repository manager contract helpers', () => { + it('should return null contract and excerpt when manager contract is absent', async () => { + const connector = { id: 'connector-1', internal_id: 'connector-1' }; + await expect(computeManagerConnectorContract({} as any, {} as any, connector as any)).resolves.toBeNull(); + await expect(computeManagerConnectorExcerpt({} as any, {} as any, connector as any)).resolves.toBeNull(); + }); + + it('should compute manager contract JSON from embedded contract snapshot', async () => { + const contract = await computeManagerConnectorContract({} as any, {} as any, connectorWithContract() as any); + expect(contract).not.toBeNull(); + const parsed = JSON.parse(contract!); + expect(parsed.slug).toBe('ipinfo'); + expect(parsed.container_image).toBe('opencti/connector-ipinfo'); + expect(parsed.container_version).toBe('1.2.3'); + }); + + it('should compute manager contract excerpt from embedded contract snapshot', async () => { + const excerpt = await computeManagerConnectorExcerpt({} as any, {} as any, connectorWithContract() as any); + expect(excerpt).toEqual({ + title: 'IPinfo', + slug: 'ipinfo', + logo: '/storage/view/catalog-logos/logo.png', + }); + }); + + it('should compute manager connector image from embedded contract snapshot', async () => { + const image = await computeManagerConnectorImage(connectorWithContract() as any); + expect(image).toBe('opencti/connector-ipinfo:1.2.3'); + }); + + it('should throw FunctionalError on invalid contract snapshot', async () => { + const connector = connectorWithContract(); + delete (connector.manager_contract as any).image; + await expect(computeManagerConnectorImage(connector as any)).rejects.toThrowError(FunctionalError('Invalid manager contract snapshot').message); + }); +}); + diff --git a/opencti-platform/opencti-graphql/tests/01-unit/domain/catalog-domain/contract-mapping-test.ts b/opencti-platform/opencti-graphql/tests/01-unit/domain/catalog-domain/contract-mapping-test.ts new file mode 100644 index 000000000000..26b7bbf85a13 --- /dev/null +++ b/opencti-platform/opencti-graphql/tests/01-unit/domain/catalog-domain/contract-mapping-test.ts @@ -0,0 +1,138 @@ +import { describe, expect, it } from 'vitest'; +import type { CatalogContract, CatalogContractEntityFields } from '../../../../src/modules/catalog/catalog-types'; +import { + mapContractDtoV0ToContractEntityFields, + mapContractEntityFieldsToEmbeddedConnectorManagerContract, + mapContractEntityFieldsToGraphqlCatalogContract, +} from '../../../../src/modules/catalog/catalog-domain'; + +const buildContractEntityFields = (): CatalogContractEntityFields => ({ + catalog_id: 'catalog-1', + contract_id: 'ipinfo-1.2.3', + content_hash: 'hash-1', + title: 'IPinfo', + slug: 'ipinfo', + description: 'desc', + short_description: 'short', + logo_uri: '/storage/view/catalog-logos/logo.png', + use_cases: ['a'], + verified: true, + last_verified_date: '2024-01-01', + playbook_supported: false, + max_confidence_level: 50, + support_version: '6.7.0', + subscription_link: null as unknown as string, + source_code: '', + manager_supported: true, + contract_version: '1.2.3', + image: 'opencti/connector-ipinfo', + connector_type: 'EXTERNAL_IMPORT', + config_schema: { + $schema: 'https://json-schema.org/draft/2020-12/schema', + $id: 'schema-id', + type: 'object', + properties: { + OPENCTI_URL: { type: 'string' }, + OPENCTI_TOKEN: { type: 'string', format: 'password' }, + USERNAME: { type: 'string' }, + }, + required: ['OPENCTI_URL', 'OPENCTI_TOKEN', 'USERNAME'], + additionalProperties: false, + }, + license_type: 'Free', + solution_categories: [], + contact: null as unknown as string, +}); + +describe('catalog contract mappings', () => { + it('should normalize V0 support_version when mapping to entity fields', () => { + const dto: CatalogContract = { + title: 'IPinfo', + slug: 'ipinfo', + description: 'desc', + short_description: 'short', + logo: null, + use_cases: [], + verified: true, + last_verified_date: '2024-01-01', + playbook_supported: false, + max_confidence_level: 50, + support_version: '>= 6.7.0', + subscription_link: null, + source_code: '', + manager_supported: true, + container_version: '1.2.3', + container_image: 'opencti/connector-ipinfo', + container_type: 'EXTERNAL_IMPORT', + config_schema: { + $schema: 'https://json-schema.org/draft/2020-12/schema', + $id: 'schema-id', + type: 'object', + properties: {}, + required: [], + additionalProperties: true, + }, + license_type: null, + solution_categories: [], + contact: null, + }; + const mapped = mapContractDtoV0ToContractEntityFields({ + catalogId: 'catalog-1', + contractDto: dto, + contractContentHash: 'hash-1', + logoUri: null, + }); + expect(mapped.contract_id).toBe('ipinfo-1.2.3'); + expect(mapped.support_version).toBe('6.7.0'); + expect(mapped.contract_version).toBe('1.2.3'); + }); + + it('should strip runtime config vars from GraphQL contract mapping when requested', () => { + const mapped = mapContractEntityFieldsToGraphqlCatalogContract(buildContractEntityFields(), { excludeRuntimeConfigVars: true }); + expect(mapped.config_schema.properties).toEqual({ + USERNAME: { type: 'string' }, + }); + expect(mapped.config_schema.required).toEqual(['USERNAME']); + }); + + it('should keep runtime config vars when GraphQL contract mapping does not request stripping', () => { + const mapped = mapContractEntityFieldsToGraphqlCatalogContract(buildContractEntityFields()); + expect(Object.keys(mapped.config_schema.properties)).toContain('OPENCTI_URL'); + expect(Object.keys(mapped.config_schema.properties)).toContain('OPENCTI_TOKEN'); + }); + + it('should normalize malformed config schema when mapping GraphQL contracts', () => { + const contract = { + ...buildContractEntityFields(), + config_schema: { + type: 'object', + }, + } as unknown as CatalogContractEntityFields; + const mapped = mapContractEntityFieldsToGraphqlCatalogContract(contract, { excludeRuntimeConfigVars: true }); + expect(mapped.config_schema).toMatchObject({ + type: 'object', + properties: {}, + required: [], + additionalProperties: true, + }); + }); + + it('should sanitize embedded manager contract snapshot to known fields only', () => { + const raw = { + ...buildContractEntityFields(), + _index: 'internal-objects', + _score: 1, + entity_type: 'CatalogContract', + } as unknown as CatalogContractEntityFields; + const embedded = mapContractEntityFieldsToEmbeddedConnectorManagerContract(raw); + expect(embedded).toMatchObject({ + catalog_id: 'catalog-1', + contract_id: 'ipinfo-1.2.3', + contract_version: '1.2.3', + image: 'opencti/connector-ipinfo', + }); + expect((embedded as any)._index).toBeUndefined(); + expect((embedded as any)._score).toBeUndefined(); + expect((embedded as any).entity_type).toBeUndefined(); + }); +}); diff --git a/opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-http-test.ts b/opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-http-test.ts new file mode 100644 index 000000000000..cc65235d8cc7 --- /dev/null +++ b/opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-http-test.ts @@ -0,0 +1,106 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const { mockCreateAuthenticatedContext, mockDownloadFile, mockSetCookieError, mockMimeTypeFromExtension } = vi.hoisted(() => ({ + mockCreateAuthenticatedContext: vi.fn(), + mockDownloadFile: vi.fn(), + mockSetCookieError: vi.fn(), + mockMimeTypeFromExtension: vi.fn(), +})); + +vi.mock('../../../../src/http/httpAuthenticatedContext', () => ({ + createAuthenticatedContext: mockCreateAuthenticatedContext, +})); + +vi.mock('../../../../src/database/raw-file-storage', () => ({ + downloadFile: mockDownloadFile, +})); + +vi.mock('../../../../src/http/httpUtils', () => ({ + setCookieError: mockSetCookieError, +})); + +vi.mock('../../../../src/modules/catalog/catalog-logo-storage', () => ({ + CATALOG_CONTRACT_LOGOS_DIR: 'catalog-logos', + getMimeTypeFromImageExtension: mockMimeTypeFromExtension, +})); + +vi.mock('../../../../src/modules/catalog/catalog-logger', () => ({ + logCatalog: { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + }, +})); + +import { handleCatalogLogoViewRequest } from '../../../../src/modules/catalog/catalog-http'; + +const buildRes = () => { + const res: any = {}; + res.status = vi.fn().mockReturnValue(res); + res.send = vi.fn().mockReturnValue(res); + res.sendStatus = vi.fn().mockReturnValue(res); + res.set = vi.fn().mockReturnValue(res); + return res; +}; + +describe('catalog-http', () => { + beforeEach(() => { + vi.clearAllMocks(); + mockMimeTypeFromExtension.mockReturnValue('image/png'); + }); + + it('should return 403 when unauthenticated', async () => { + mockCreateAuthenticatedContext.mockResolvedValue({ user: null, source: 'catalog_logo_view' }); + const req = { params: { file: 'abc.png' } } as any; + const res = buildRes(); + await handleCatalogLogoViewRequest(req, res, vi.fn()); + expect(res.sendStatus).toHaveBeenCalledWith(403); + }); + + it('should return 404 when logo does not exist', async () => { + mockCreateAuthenticatedContext.mockResolvedValue({ user: { id: 'u1' }, source: 'catalog_logo_view' }); + mockDownloadFile.mockResolvedValue(null); + const req = { params: { file: 'abc.png' } } as any; + const res = buildRes(); + await handleCatalogLogoViewRequest(req, res, vi.fn()); + expect(res.status).toHaveBeenCalledWith(404); + expect(res.send).toHaveBeenCalledWith({ status: 'error', error: 'Catalog logo not found' }); + }); + + it('should stream file and set expected headers', async () => { + const pipe = vi.fn(); + mockCreateAuthenticatedContext.mockResolvedValue({ user: { id: 'u1' }, source: 'catalog_logo_view' }); + mockDownloadFile.mockResolvedValue({ pipe }); + const req = { params: { file: 'abc.png' } } as any; + const res = buildRes(); + await handleCatalogLogoViewRequest(req, res, vi.fn()); + expect(res.set).toHaveBeenCalledWith({ 'Content-Security-Policy': 'sandbox' }); + expect(res.set).toHaveBeenCalledWith('Cache-Control', 'private, no-cache, no-store, must-revalidate'); + expect(res.set).toHaveBeenCalledWith({ Pragma: 'no-cache' }); + expect(res.set).toHaveBeenCalledWith('Content-type', 'image/png'); + expect(pipe).toHaveBeenCalledWith(res); + }); + + it('should handle invalid URL parameters', async () => { + mockCreateAuthenticatedContext.mockResolvedValue({ user: { id: 'u1' }, source: 'catalog_logo_view' }); + const req = { params: {} } as any; + const res = buildRes(); + await handleCatalogLogoViewRequest(req, res, vi.fn()); + expect(mockSetCookieError).toHaveBeenCalled(); + expect(res.status).toHaveBeenCalledWith(503); + expect(res.send).toHaveBeenCalledWith({ status: 'error', error: 'Invalid URL format' }); + }); + + it('should handle download errors with 503', async () => { + mockCreateAuthenticatedContext.mockResolvedValue({ user: { id: 'u1' }, source: 'catalog_logo_view' }); + mockDownloadFile.mockRejectedValue(new Error('storage failure')); + const req = { params: { file: 'abc.png' } } as any; + const res = buildRes(); + await handleCatalogLogoViewRequest(req, res, vi.fn()); + expect(mockSetCookieError).toHaveBeenCalledWith(res, 'storage failure'); + expect(res.status).toHaveBeenCalledWith(503); + expect(res.send).toHaveBeenCalledWith({ status: 'error', error: 'storage failure' }); + }); +}); + diff --git a/opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-logo-storage-test.ts b/opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-logo-storage-test.ts new file mode 100644 index 000000000000..4e83f54ee741 --- /dev/null +++ b/opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-logo-storage-test.ts @@ -0,0 +1,128 @@ +import { describe, expect, it } from 'vitest'; +import type { CatalogContract } from '../../../../src/modules/catalog/catalog-types'; +import { computeCatalogContractLogoUploadOperation, getMimeTypeFromImageExtension } from '../../../../src/modules/catalog/catalog-logo-storage'; + +const buildContract = (logo: string | null): CatalogContract => ({ + title: 'Contract title', + slug: 'contract-slug', + description: 'description', + short_description: 'short description', + logo, + use_cases: [], + verified: true, + last_verified_date: '2024-01-01', + playbook_supported: false, + max_confidence_level: 50, + support_version: '6.7.0', + subscription_link: null, + source_code: '', + manager_supported: true, + container_version: '1.2.3', + container_image: 'opencti/connector-test', + container_type: 'EXTERNAL_IMPORT', + config_schema: { + $schema: 'https://json-schema.org/draft/2020-12/schema', + $id: 'id', + type: 'object', + properties: {}, + required: [], + additionalProperties: true, + }, + license_type: null, + solution_categories: [], + contact: null, +}); + +describe('catalog-logo-storage', () => { + it('should return no-logo when contract has no logo', () => { + const result = computeCatalogContractLogoUploadOperation(buildContract(null), new Set()); + expect(result).toEqual({ result: 'no-logo', logoUri: null }); + }); + + it('should reject non data URL logos', () => { + const result = computeCatalogContractLogoUploadOperation(buildContract('https://example.org/logo.png'), new Set()); + expect(result.result).toBe('failed'); + if (result.result === 'failed') { + expect(result.error.message).toContain('not a data URL'); + } + }); + + it('should reject unsupported mime type', () => { + const result = computeCatalogContractLogoUploadOperation(buildContract('data:text/plain;base64,Zm9v'), new Set()); + expect(result.result).toBe('failed'); + if (result.result === 'failed') { + expect(result.error.message).toContain('Unsupported logo mime type'); + } + }); + + it('should reject invalid base64 payload', () => { + const result = computeCatalogContractLogoUploadOperation(buildContract('data:image/png;base64,not-valid-***'), new Set()); + expect(result.result).toBe('failed'); + if (result.result === 'failed') { + expect(result.error.message).toContain('invalid base64 payload'); + } + }); + + it('should reject invalid URL-encoded payload', () => { + const result = computeCatalogContractLogoUploadOperation(buildContract('data:image/svg+xml,%E0%A4%A'), new Set()); + expect(result.result).toBe('failed'); + if (result.result === 'failed') { + expect(result.error.message).toContain('invalid URL-encoded payload'); + } + }); + + it('should reject zero-length payload', () => { + const result = computeCatalogContractLogoUploadOperation(buildContract('data:image/svg+xml,'), new Set()); + expect(result.result).toBe('failed'); + if (result.result === 'failed') { + expect(result.error.message).toContain('Unexpected zero-length logo data'); + } + }); + + it('should return dedup success when logo already exists', () => { + const first = computeCatalogContractLogoUploadOperation(buildContract('data:image/png;base64,Zm9v'), new Set()); + expect(first.result).toBe('success'); + if (first.result !== 'success') { + return; + } + const second = computeCatalogContractLogoUploadOperation( + buildContract('data:image/png;base64,Zm9v'), + new Set([first.filename]), + ); + expect(second).toEqual({ + result: 'success', + existed: true, + logoUri: first.logoUri, + filename: first.filename, + }); + }); + + it('should build upload operation for base64 payload', () => { + const result = computeCatalogContractLogoUploadOperation(buildContract('data:image/png;base64,Zm9v'), new Set()); + expect(result.result).toBe('success'); + if (result.result !== 'success') { + return; + } + expect(result.existed).toBe(false); + expect(result.filename.endsWith('.png')).toBe(true); + expect(result.logoUri).toContain('/catalog/logo/'); + expect(result.operation?.s3Key).toContain('catalog-logos/'); + expect(result.operation?.body.toString('utf8')).toBe('foo'); + }); + + it('should build upload operation for URL-encoded payload', () => { + const result = computeCatalogContractLogoUploadOperation(buildContract('data:image/svg+xml,%3Csvg%3Eok%3C%2Fsvg%3E'), new Set()); + expect(result.result).toBe('success'); + if (result.result !== 'success') { + return; + } + expect(result.filename.endsWith('.svg')).toBe(true); + expect(result.operation?.body.toString('utf8')).toBe('ok'); + }); + + it('should resolve mime type from extension', () => { + expect(getMimeTypeFromImageExtension('.png')).toBe('image/png'); + expect(getMimeTypeFromImageExtension('.unknown')).toBeUndefined(); + }); +}); + diff --git a/opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-sync-domain-test.ts b/opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-sync-domain-test.ts new file mode 100644 index 000000000000..b25a16cbd3e7 --- /dev/null +++ b/opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-sync-domain-test.ts @@ -0,0 +1,232 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { pathToFileURL } from 'node:url'; + +const { + mockFetchSourceCatalog, + mockFetchSourceCatalogRevisionHint, + mockFindCatalogBySourceUri, + mockFindCatalogByCatalogId, + mockFindCatalogContractsByCatalogId, + mockFindCatalogs, + mockInsertCatalogContracts, + mockUpdateCatalogContracts, + mockDeleteCatalogContracts, + mockUpsertCatalog, + mockDeleteCatalogs, + mockListCatalogContractLogos, + mockConfGet, + featureFlagState, +} = vi.hoisted(() => ({ + mockFetchSourceCatalog: vi.fn(), + mockFetchSourceCatalogRevisionHint: vi.fn(), + mockFindCatalogBySourceUri: vi.fn(), + mockFindCatalogByCatalogId: vi.fn(), + mockFindCatalogContractsByCatalogId: vi.fn(), + mockFindCatalogs: vi.fn(), + mockInsertCatalogContracts: vi.fn(), + mockUpdateCatalogContracts: vi.fn(), + mockDeleteCatalogContracts: vi.fn(), + mockUpsertCatalog: vi.fn(), + mockDeleteCatalogs: vi.fn(), + mockListCatalogContractLogos: vi.fn(), + mockConfGet: vi.fn((key: string) => { + if (key === 'redis:ca') return []; + if (key === 'redis:use_ssl') return false; + return undefined; + }), + featureFlagState: { value: false }, +})); + +vi.mock('../../../../src/config/conf', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + default: { + ...actual.default, + get: mockConfGet, + }, + isFeatureEnabled: vi.fn(() => featureFlagState.value), + PLATFORM_VERSION: '7.2.0', + }; +}); + +vi.mock('../../../../src/utils/access', () => ({ + SYSTEM_USER: { id: 'system-user' }, +})); + +vi.mock('../../../../src/modules/catalog/sync/catalog-sync-source-gateway', () => ({ + fetchSourceCatalog: mockFetchSourceCatalog, + fetchSourceCatalogRevisionHint: mockFetchSourceCatalogRevisionHint, +})); + +vi.mock('../../../../src/modules/catalog/catalog-repository', () => ({ + findCatalogBySourceUri: mockFindCatalogBySourceUri, + findCatalogByCatalogId: mockFindCatalogByCatalogId, + findCatalogContractsByCatalogId: mockFindCatalogContractsByCatalogId, + findCatalogs: mockFindCatalogs, + insertCatalogContracts: mockInsertCatalogContracts, + updateCatalogContracts: mockUpdateCatalogContracts, + deleteCatalogContracts: mockDeleteCatalogContracts, + upsertCatalog: mockUpsertCatalog, + deleteCatalogs: mockDeleteCatalogs, +})); + +vi.mock('../../../../src/modules/catalog/catalog-logo-storage', () => ({ + listCatalogContractLogos: mockListCatalogContractLogos, + computeCatalogContractLogoUploadOperation: vi.fn(() => ({ result: 'no-logo', logoUri: null })), + uploadCatalogContractLogoOperation: vi.fn(async () => ({ result: 'success' })), +})); + +vi.mock('../../../../src/modules/catalog/catalog-logger', () => ({ + logCatalog: { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + }, +})); + +vi.mock('../../../../src/schema/identifier', () => ({ + idGenFromData: vi.fn((entityType: string, data: Record) => `${entityType}-${Object.values(data).join('-')}-internal`), + generateStandardId: vi.fn((entityType: string, data: Record) => `${entityType}-${Object.values(data).join('-')}-standard`), +})); + +import { synchronizeCatalogs } from '../../../../src/modules/catalog/sync/catalog-sync-domain'; + +const buildSourceCatalog = (id: string) => ({ + id, + name: `Catalog ${id}`, + description: 'desc', + product_version: '7.2.0', + manifest_version: null, + manifest_schema_version: '0' as const, + contracts: [], +}); + +describe('catalog-sync-domain', () => { + beforeEach(() => { + vi.clearAllMocks(); + featureFlagState.value = false; + mockConfGet.mockImplementation((key: string) => { + if (key === 'redis:ca') return []; + if (key === 'redis:use_ssl') return false; + if (key === 'app:custom_catalogs') return []; + if (key === 'catalog_manager:custom_catalog_refresh_endpoint_uri') return undefined; + if (key === 'xtm:xtmhub_url') return undefined; + return undefined; + }); + mockFetchSourceCatalogRevisionHint.mockResolvedValue(undefined); + mockFindCatalogBySourceUri.mockResolvedValue(undefined); + mockFindCatalogByCatalogId.mockResolvedValue(undefined); + mockFindCatalogContractsByCatalogId.mockResolvedValue(new Map()); + mockFindCatalogs.mockResolvedValue([]); + mockListCatalogContractLogos.mockResolvedValue(new Set()); + mockInsertCatalogContracts.mockResolvedValue(undefined); + mockUpdateCatalogContracts.mockResolvedValue(undefined); + mockDeleteCatalogContracts.mockResolvedValue(undefined); + mockUpsertCatalog.mockResolvedValue(undefined); + mockDeleteCatalogs.mockResolvedValue(undefined); + }); + + it('should use embedded source by default', async () => { + mockFetchSourceCatalog.mockResolvedValue(buildSourceCatalog('embedded-catalog')); + const result = await synchronizeCatalogs({ source: 'test' } as any, { id: 'user-1' } as any); + expect(result).toEqual(['embedded-catalog']); + expect(mockFetchSourceCatalog).toHaveBeenCalledWith({ kind: 'embedded', uri: 'embedded' }, { source: 'test' }); + expect(mockFindCatalogs).toHaveBeenCalled(); + }); + + it('should include custom sources and deduplicate by uri', async () => { + mockConfGet.mockImplementation((key: string) => { + if (key === 'redis:ca') return []; + if (key === 'redis:use_ssl') return false; + if (key === 'catalog_manager:custom_catalog_refresh_endpoint_uri') return 'https://catalog.example.org/custom.json'; + if (key === 'app:custom_catalogs') return ['https://catalog.example.org/custom.json', '/tmp/custom-catalog.json']; + if (key === 'xtm:xtmhub_url') return undefined; + return undefined; + }); + mockFetchSourceCatalog.mockImplementation(async (source: { uri: string }) => buildSourceCatalog(source.uri)); + const result = await synchronizeCatalogs({ source: 'test' } as any, { id: 'user-1' } as any); + const calledSources = mockFetchSourceCatalog.mock.calls.map((call) => call[0]); + expect(calledSources).toEqual([ + { kind: 'embedded', uri: 'embedded' }, + { kind: 'remote', uri: 'https://catalog.example.org/custom.json' }, + { kind: 'local', filepath: '/tmp/custom-catalog.json', uri: pathToFileURL('/tmp/custom-catalog.json').toString() }, + ]); + expect(result).toHaveLength(3); + }); + + it('should ignore custom catalog manager endpoint when decoupling feature is disabled', async () => { + mockConfGet.mockImplementation((key: string) => { + if (key === 'redis:ca') return []; + if (key === 'redis:use_ssl') return false; + if (key === 'catalog_manager:custom_catalog_refresh_endpoint_uri') return 'https://catalog.example.org/custom.json'; + if (key === 'app:custom_catalogs') return []; + if (key === 'xtm:xtmhub_url') return 'https://xtm.example.org'; + return undefined; + }); + mockFetchSourceCatalog.mockResolvedValue(buildSourceCatalog('embedded-catalog')); + await synchronizeCatalogs({ source: 'test' } as any, { id: 'user-1' } as any); + expect(mockFetchSourceCatalog).toHaveBeenCalledTimes(1); + expect(mockFetchSourceCatalog).toHaveBeenCalledWith({ kind: 'embedded', uri: 'embedded' }, { source: 'test' }); + }); + + it('should use custom catalog manager endpoint instead of xtm hub source when decoupling feature is enabled', async () => { + featureFlagState.value = true; + mockConfGet.mockImplementation((key: string) => { + if (key === 'redis:ca') return []; + if (key === 'redis:use_ssl') return false; + if (key === 'catalog_manager:custom_catalog_refresh_endpoint_uri') return 'https://catalog.example.org/custom.json'; + if (key === 'app:custom_catalogs') return []; + if (key === 'xtm:xtmhub_url') return 'https://xtm.example.org'; + return undefined; + }); + mockFetchSourceCatalog.mockResolvedValue(buildSourceCatalog('custom-catalog')); + await synchronizeCatalogs({ source: 'test' } as any, { id: 'user-1' } as any); + expect(mockFetchSourceCatalog).toHaveBeenCalledTimes(1); + expect(mockFetchSourceCatalog).toHaveBeenCalledWith({ kind: 'remote', uri: 'https://catalog.example.org/custom.json' }, { source: 'test' }); + }); + + it('should fallback to embedded when remote decoupling source fails before first persistence', async () => { + featureFlagState.value = true; + mockConfGet.mockImplementation((key: string) => { + if (key === 'redis:ca') return []; + if (key === 'redis:use_ssl') return false; + if (key === 'xtm:xtmhub_url') return 'https://xtm.example.org'; + if (key === 'app:custom_catalogs') return []; + if (key === 'catalog_manager:custom_catalog_refresh_endpoint_uri') return undefined; + return undefined; + }); + mockFetchSourceCatalog.mockImplementation(async (source: { kind: string }) => { + if (source.kind === 'remote') { + throw new Error('remote unavailable'); + } + return buildSourceCatalog('embedded-fallback'); + }); + const result = await synchronizeCatalogs({ source: 'test' } as any, { id: 'user-1' } as any); + expect(result).toEqual(['embedded-fallback']); + expect(mockFetchSourceCatalog.mock.calls[0][0].kind).toBe('remote'); + expect(mockFetchSourceCatalog.mock.calls[1][0]).toEqual({ kind: 'embedded', uri: 'embedded' }); + }); + + it('should skip obsolete catalog cleanup when at least one source sync fails', async () => { + mockConfGet.mockImplementation((key: string) => { + if (key === 'redis:ca') return []; + if (key === 'redis:use_ssl') return false; + if (key === 'catalog_manager:custom_catalog_refresh_endpoint_uri') return undefined; + if (key === 'app:custom_catalogs') return ['/tmp/custom-catalog.json']; + if (key === 'xtm:xtmhub_url') return undefined; + return undefined; + }); + mockFetchSourceCatalog.mockImplementation(async (source: { kind: string }) => { + if (source.kind === 'local') { + throw new Error('local source failure'); + } + return buildSourceCatalog('embedded-catalog'); + }); + const result = await synchronizeCatalogs({ source: 'test' } as any, { id: 'user-1' } as any); + expect(result).toEqual(['embedded-catalog']); + expect(mockFindCatalogs).not.toHaveBeenCalled(); + expect(mockDeleteCatalogs).not.toHaveBeenCalled(); + }); +}); diff --git a/opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-sync-source-gateway-test.ts b/opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-sync-source-gateway-test.ts new file mode 100644 index 000000000000..6e277ef7b39c --- /dev/null +++ b/opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-sync-source-gateway-test.ts @@ -0,0 +1,228 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const { mockReadFile, mockGet, mockHead, mockGetOrCompileValidator } = vi.hoisted(() => ({ + mockReadFile: vi.fn(), + mockGet: vi.fn(), + mockHead: vi.fn(), + mockGetOrCompileValidator: vi.fn(), +})); + +vi.mock('node:fs/promises', () => ({ + readFile: mockReadFile, +})); + +vi.mock('../../../../src/config/conf', () => ({ + default: { + get: vi.fn(() => undefined), + }, + TEST_MODE: true, + ENABLED_METRICS: false, + booleanConf: vi.fn(() => false), + loadCert: vi.fn(() => ''), +})); + +vi.mock('../../../../src/database/utils', () => ({ + isEmptyField: (value: unknown) => value === null || value === undefined || value === '', +})); + +vi.mock('../../../../src/modules/catalog/catalog-domain', () => ({ + getOrCompileValidator: mockGetOrCompileValidator, +})); + +vi.mock('../../../../src/utils/http-client', () => ({ + getHttpClient: vi.fn(() => ({ + get: mockGet, + head: mockHead, + })), +})); + +vi.mock('../../../../src/modules/catalog/catalog-logger', () => ({ + logCatalog: { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + }, +})); + +import { + fetchSourceCatalog, + fetchSourceCatalogRevisionHint, + mapCatalogContractDtoToCatalogContractSyncSource, +} from '../../../../src/modules/catalog/sync/catalog-sync-source-gateway'; + +const baseV0Contract = { + title: 'IPinfo', + slug: 'ipinfo', + description: 'desc', + short_description: 'short', + logo: 'data:image/png;base64,Zm9v', + use_cases: [], + verified: true, + last_verified_date: '2024-01-01', + playbook_supported: false, + max_confidence_level: 50, + support_version: '>= 6.7.0', + subscription_link: null, + source_code: '', + manager_supported: true, + container_version: '1.2.3', + container_image: 'opencti/connector-ipinfo', + container_type: 'EXTERNAL_IMPORT', + config_schema: { + $schema: 'https://json-schema.org/draft/2020-12/schema', + $id: 'id', + type: 'object', + properties: {}, + required: [], + additionalProperties: true, + }, + license_type: null, + solution_categories: [], + contact: null, +}; + +const baseV1Contract = { + id: 'ipinfo-1.2.3', + title: 'IPinfo', + slug: 'ipinfo', + description: 'desc', + short_description: 'short', + logo: 'data:image/png;base64,Zm9v', + use_cases: [], + verified: true, + last_verified_date: '2024-01-01', + subscription_link: null, + source_code: '', + manager_supported: true, + support_version: '6.7.0', + license_type: null, + contact: null, + solution_categories: [], + version: '1.2.3', + image_name: 'opencti/connector-ipinfo', + image_type: 'EXTERNAL_IMPORT', + additional_properties: { + playbook_supported: true, + max_confidence_level: 80, + }, + config_schema: { + type: 'object', + properties: {}, + required: [], + additionalProperties: true, + }, +}; + +describe('catalog-sync-source-gateway', () => { + beforeEach(() => { + vi.clearAllMocks(); + mockGetOrCompileValidator.mockReturnValue({}); + }); + + it('should normalize support_version and id from V0 contract DTO', () => { + const mapped = mapCatalogContractDtoToCatalogContractSyncSource(baseV0Contract as any); + expect(mapped.id).toBe('ipinfo-1.2.3'); + expect(mapped.support_version).toBe('6.7.0'); + }); + + it('should map V0 catalog sources', async () => { + mockReadFile.mockResolvedValue(JSON.stringify({ + id: 'filigran', + name: 'Filigran catalog', + description: 'catalog', + version: '6.8.0', + contracts: [baseV0Contract], + })); + const source = await fetchSourceCatalog({ + kind: 'local', + filepath: '/tmp/catalog.json', + uri: 'file:///tmp/catalog.json', + }); + expect(source.manifest_schema_version).toBe('0'); + expect(source.manifest_version).toBeNull(); + expect(source.product_version).toBe('6.8.0'); + expect(source.contracts).toHaveLength(1); + expect(source.contracts[0].support_version).toBe('6.7.0'); + }); + + it('should map V1 catalog sources', async () => { + mockReadFile.mockResolvedValue(JSON.stringify({ + id: 'filigran', + name: 'Filigran catalog', + description: 'catalog', + manifest_schema_version: '1', + manifest_version: '2026.08', + product_version: '6.8.0', + contracts: [baseV1Contract], + })); + const source = await fetchSourceCatalog({ + kind: 'local', + filepath: '/tmp/catalog-v1.json', + uri: 'file:///tmp/catalog-v1.json', + }); + expect(source.manifest_schema_version).toBe('1'); + expect(source.manifest_version).toBe('2026.08'); + expect(source.product_version).toBe('6.8.0'); + expect(source.contracts[0].id).toBe('ipinfo-1.2.3'); + expect(source.contracts[0].container_image).toBe('opencti/connector-ipinfo'); + }); + + it('should reject unsupported schema versions', async () => { + mockReadFile.mockResolvedValue(JSON.stringify({ + id: 'filigran', + name: 'Filigran catalog', + description: 'catalog', + manifest_schema_version: '2', + contracts: [], + })); + await expect(fetchSourceCatalog({ + kind: 'local', + filepath: '/tmp/catalog-invalid.json', + uri: 'file:///tmp/catalog-invalid.json', + })).rejects.toThrowError('Unsupported catalog schema version'); + }); + + it('should reject unrecognized formats', async () => { + mockReadFile.mockResolvedValue(JSON.stringify({ hello: 'world' })); + await expect(fetchSourceCatalog({ + kind: 'local', + filepath: '/tmp/catalog-invalid.json', + uri: 'file:///tmp/catalog-invalid.json', + })).rejects.toThrowError('Unrecognized catalog format'); + }); + + it('should reject manager-supported contracts missing container image', async () => { + const invalidContract = { ...baseV0Contract, container_image: '' }; + mockReadFile.mockResolvedValue(JSON.stringify({ + id: 'filigran', + name: 'Filigran catalog', + description: 'catalog', + version: '6.8.0', + contracts: [invalidContract], + })); + await expect(fetchSourceCatalog({ + kind: 'local', + filepath: '/tmp/catalog-invalid-contract.json', + uri: 'file:///tmp/catalog-invalid-contract.json', + })).rejects.toThrowError('Contract must define container_image field'); + }); + + it('should return revision hint from remote ETag header', async () => { + mockHead.mockResolvedValue({ headers: { etag: '"abc123"' } }); + const hint = await fetchSourceCatalogRevisionHint({ + kind: 'remote', + uri: 'https://catalog.example.org/manifest.json', + }); + expect(hint).toBe('"abc123"'); + }); + + it('should return undefined revision hint when ETag is missing', async () => { + mockHead.mockResolvedValue({ headers: {} }); + const hint = await fetchSourceCatalogRevisionHint({ + kind: 'remote', + uri: 'https://catalog.example.org/manifest.json', + }); + expect(hint).toBeUndefined(); + }); +}); diff --git a/opencti-platform/opencti-graphql/tests/01-unit/modules/connector/connector-domain-auto-upgrade-test.ts b/opencti-platform/opencti-graphql/tests/01-unit/modules/connector/connector-domain-auto-upgrade-test.ts new file mode 100644 index 000000000000..8162344dd348 --- /dev/null +++ b/opencti-platform/opencti-graphql/tests/01-unit/modules/connector/connector-domain-auto-upgrade-test.ts @@ -0,0 +1,189 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type { BasicStoreEntityConnector } from '../../../../src/types/connector'; + +const { + mockFindManagedConnectorsByCatalogId, + mockFindLatestCompatibleCatalogContractBySlug, + mockMapContractEntityFieldsToEmbeddedConnectorManagerContract, + mockPatchAttribute, + mockPublishUserAction, +} = vi.hoisted(() => ({ + mockFindManagedConnectorsByCatalogId: vi.fn(), + mockFindLatestCompatibleCatalogContractBySlug: vi.fn(), + mockMapContractEntityFieldsToEmbeddedConnectorManagerContract: vi.fn((c) => ({ ...c })), + mockPatchAttribute: vi.fn(), + mockPublishUserAction: vi.fn(), +})); + +vi.mock('../../../../src/modules/connector/connector-repository', () => ({ + findManagedConnectorsByCatalogId: mockFindManagedConnectorsByCatalogId, +})); + +vi.mock('../../../../src/modules/catalog/catalog-repository', () => ({ + findLatestCompatibleCatalogContractBySlug: mockFindLatestCompatibleCatalogContractBySlug, +})); + +vi.mock('../../../../src/modules/catalog/catalog-domain', () => ({ + mapContractEntityFieldsToEmbeddedConnectorManagerContract: mockMapContractEntityFieldsToEmbeddedConnectorManagerContract, +})); + +vi.mock('../../../../src/database/middleware', () => ({ + patchAttribute: mockPatchAttribute, +})); + +vi.mock('../../../../src/listener/UserActionListener', () => ({ + publishUserAction: mockPublishUserAction, +})); + +vi.mock('../../../../src/config/conf', () => ({ + logApp: { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + }, +})); + +import { autoUpgradeManagedConnectors } from '../../../../src/modules/connector/connector-domain'; + +const buildManagedConnector = (overrides: Partial = {}): BasicStoreEntityConnector => ({ + id: 'connector-1', + internal_id: 'connector-1', + name: 'Managed connector', + title: 'Managed connector', + connector_type: 'EXTERNAL_IMPORT', + active: true, + auto: true, + built_in: false, + only_contextual: false, + connector_scope: [], + updated_at: '', + connector_user_id: 'user-1', + connector_info: {} as any, + playbook_compatible: false, + xtm_one_intent: null, + connector_state: '', + connector_state_reset: false, + connector_trigger_filters: '', + catalog_id: 'catalog-1', + manager_upgrade_strategy: 'latest', + manager_contract_image: 'opencti/connector-test:1.0.0', + manager_contract: { + slug: 'ipinfo', + contract_version: '1.0.0', + content_hash: 'hash-1', + } as any, + ...overrides, +}); + +const latestCompatibleContract = { + slug: 'ipinfo', + contract_version: '1.1.0', + content_hash: 'hash-2', + image: 'opencti/connector-test', +}; + +describe('connector-domain auto-upgrade', () => { + beforeEach(() => { + vi.clearAllMocks(); + mockFindManagedConnectorsByCatalogId.mockResolvedValue([]); + mockFindLatestCompatibleCatalogContractBySlug.mockResolvedValue(undefined); + }); + + it('should skip connectors when strategy is not latest', async () => { + mockFindManagedConnectorsByCatalogId.mockResolvedValue([ + buildManagedConnector({ manager_upgrade_strategy: 'manual' as any }), + ]); + await autoUpgradeManagedConnectors({ source: 'test' } as any, { id: 'user-1' } as any, ['catalog-1']); + expect(mockPatchAttribute).not.toHaveBeenCalled(); + }); + + it('should skip connectors without manager_contract snapshot', async () => { + mockFindManagedConnectorsByCatalogId.mockResolvedValue([ + buildManagedConnector({ manager_contract: undefined }), + ]); + await autoUpgradeManagedConnectors({ source: 'test' } as any, { id: 'user-1' } as any, ['catalog-1']); + expect(mockFindLatestCompatibleCatalogContractBySlug).not.toHaveBeenCalled(); + expect(mockPatchAttribute).not.toHaveBeenCalled(); + }); + + it('should skip patch when connector already has latest compatible contract', async () => { + mockFindManagedConnectorsByCatalogId.mockResolvedValue([ + buildManagedConnector({ + manager_contract: { + slug: 'ipinfo', + contract_version: '1.1.0', + content_hash: 'hash-2', + } as any, + }), + ]); + mockFindLatestCompatibleCatalogContractBySlug.mockResolvedValue(latestCompatibleContract); + await autoUpgradeManagedConnectors({ source: 'test' } as any, { id: 'user-1' } as any, ['catalog-1']); + expect(mockPatchAttribute).not.toHaveBeenCalled(); + }); + + it('should patch managed connector when a newer compatible contract exists', async () => { + mockFindManagedConnectorsByCatalogId.mockResolvedValue([buildManagedConnector()]); + mockFindLatestCompatibleCatalogContractBySlug.mockResolvedValue(latestCompatibleContract); + await autoUpgradeManagedConnectors({ source: 'test' } as any, { id: 'user-1' } as any, ['catalog-1']); + expect(mockMapContractEntityFieldsToEmbeddedConnectorManagerContract).toHaveBeenCalledWith(latestCompatibleContract); + expect(mockPatchAttribute).toHaveBeenCalledWith( + { source: 'test' }, + { id: 'user-1' }, + 'connector-1', + 'Connector', + expect.objectContaining({ + manager_contract_image: 'opencti/connector-test', + }), + ); + expect(mockPublishUserAction).toHaveBeenCalled(); + }); + + it('should patch managed connector when latest compatible contract is an older version (downgrade)', async () => { + mockFindManagedConnectorsByCatalogId.mockResolvedValue([ + buildManagedConnector({ + manager_contract: { + slug: 'ipinfo', + contract_version: '2.0.0', + content_hash: 'hash-newer', + } as any, + }), + ]); + mockFindLatestCompatibleCatalogContractBySlug.mockResolvedValue({ + ...latestCompatibleContract, + contract_version: '1.9.0', + content_hash: 'hash-older', + }); + await autoUpgradeManagedConnectors({ source: 'test' } as any, { id: 'user-1' } as any, ['catalog-1']); + expect(mockPatchAttribute).toHaveBeenCalled(); + expect(mockPublishUserAction).toHaveBeenCalled(); + }); + + it('should patch managed connector when version is equal but content hash differs', async () => { + mockFindManagedConnectorsByCatalogId.mockResolvedValue([ + buildManagedConnector({ + manager_contract: { + slug: 'ipinfo', + contract_version: '1.1.0', + content_hash: 'old-hash', + } as any, + }), + ]); + mockFindLatestCompatibleCatalogContractBySlug.mockResolvedValue({ + ...latestCompatibleContract, + contract_version: '1.1.0', + content_hash: 'new-hash', + }); + await autoUpgradeManagedConnectors({ source: 'test' } as any, { id: 'user-1' } as any, ['catalog-1']); + expect(mockPatchAttribute).toHaveBeenCalled(); + expect(mockPublishUserAction).toHaveBeenCalled(); + }); + + it('should skip patch when no compatible contract can be found', async () => { + mockFindManagedConnectorsByCatalogId.mockResolvedValue([buildManagedConnector()]); + mockFindLatestCompatibleCatalogContractBySlug.mockResolvedValue(undefined); + await autoUpgradeManagedConnectors({ source: 'test' } as any, { id: 'user-1' } as any, ['catalog-1']); + expect(mockPatchAttribute).not.toHaveBeenCalled(); + expect(mockPublishUserAction).not.toHaveBeenCalled(); + }); +}); diff --git a/opencti-platform/opencti-graphql/tests/01-unit/utils/data-url-test.ts b/opencti-platform/opencti-graphql/tests/01-unit/utils/data-url-test.ts new file mode 100644 index 000000000000..8b37f956150c --- /dev/null +++ b/opencti-platform/opencti-graphql/tests/01-unit/utils/data-url-test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from 'vitest'; +import { parseDataUrl } from '../../../src/utils/data-url'; + +describe('parseDataUrl', () => { + it('should parse mime type and base64 metadata', () => { + const parsed = parseDataUrl('data:image/png;base64,Zm9v'); + expect(parsed).toEqual({ + mimeType: 'image/png', + base64Encoded: true, + data: 'Zm9v', + }); + }); + + it('should parse metadata case-insensitively for base64 flag', () => { + const parsed = parseDataUrl('data:image/png;BASE64,Zm9v'); + expect(parsed).toEqual({ + mimeType: 'image/png', + base64Encoded: true, + data: 'Zm9v', + }); + }); + + it('should parse URL-encoded payload without base64 flag', () => { + const parsed = parseDataUrl('data:text/plain,hello%20world'); + expect(parsed).toEqual({ + mimeType: 'text/plain', + base64Encoded: false, + data: 'hello%20world', + }); + }); + + it('should fallback when no data URL prefix is present', () => { + expect(parseDataUrl('https://example.org/logo.png')).toEqual({ + mimeType: '', + base64Encoded: false, + data: '', + }); + }); + + it('should fallback when metadata/data separator is missing', () => { + expect(parseDataUrl('data:image/png;base64')).toEqual({ + mimeType: '', + base64Encoded: false, + data: '', + }); + }); +}); + diff --git a/opencti-platform/opencti-graphql/tests/01-unit/utils/logger-test.ts b/opencti-platform/opencti-graphql/tests/01-unit/utils/logger-test.ts index d13ce6b68f62..b85acde83b86 100644 --- a/opencti-platform/opencti-graphql/tests/01-unit/utils/logger-test.ts +++ b/opencti-platform/opencti-graphql/tests/01-unit/utils/logger-test.ts @@ -1,6 +1,6 @@ import * as R from 'ramda'; import { describe, expect, it } from 'vitest'; -import { appLogLevelMaxArraySize, appLogLevelMaxStringSize, prepareLogMetadata } from '../../../src/config/conf'; +import { appLogLevelMaxArraySize, appLogLevelMaxStringSize, logApp, prepareLogMetadata } from '../../../src/config/conf'; import { FunctionalError } from '../../../src/config/errors'; // region objects definition @@ -116,6 +116,15 @@ const WITH_ERROR_OBJECT = { // endregion describe('Logger test suite', () => { + it('should expose child logger helpers', () => { + const child = logApp.child({ module: 'catalog', source: 'backend' }); + expect(child).toBeDefined(); + expect(typeof child.info).toBe('function'); + expect(typeof child.warn).toBe('function'); + expect(typeof child.error).toBe('function'); + expect(typeof child.debug).toBe('function'); + }); + it('Log object is correctly untouched', () => { const cleanObject = prepareLogMetadata(CLASSIC_OBJECT); const classicCompare = R.dissoc('version', CLASSIC_OBJECT); diff --git a/opencti-platform/opencti-graphql/tests/01-unit/utils/sorting-test.ts b/opencti-platform/opencti-graphql/tests/01-unit/utils/sorting-test.ts index 7d176fbe4702..f73ff9655200 100644 --- a/opencti-platform/opencti-graphql/tests/01-unit/utils/sorting-test.ts +++ b/opencti-platform/opencti-graphql/tests/01-unit/utils/sorting-test.ts @@ -30,6 +30,14 @@ describe('Sorting utilities', () => { }, }); + sorting = await buildElasticSortingForAttributeCriteria(testContext, SYSTEM_USER, 'support_version', 'asc'); + expect(sorting).toEqual({ + support_version: { + missing: '_last', + order: 'asc', + }, + }); + // complex object with sortBy sorting = await buildElasticSortingForAttributeCriteria(testContext, SYSTEM_USER, 'group_confidence_level', 'asc'); expect(sorting).toEqual({ diff --git a/opencti-platform/opencti-graphql/tests/03-integration/02-resolvers/connector-composer-test.ts b/opencti-platform/opencti-graphql/tests/03-integration/02-resolvers/connector-composer-test.ts index 245cf03dcaad..e272d4229f9b 100644 --- a/opencti-platform/opencti-graphql/tests/03-integration/02-resolvers/connector-composer-test.ts +++ b/opencti-platform/opencti-graphql/tests/03-integration/02-resolvers/connector-composer-test.ts @@ -1362,6 +1362,14 @@ describe('Connector Composer and Managed Connectors', () => { }); expect(definition.config_schema).toBeDefined(); expect(definition.config_schema.properties).toBeDefined(); + const runtimeKeys = ['OPENCTI_TOKEN', 'OPENCTI_URL', 'CONNECTOR_TYPE', 'CONNECTOR_RUN_AND_TERMINATE']; + runtimeKeys.forEach((runtimeKey) => { + expect(definition.config_schema.properties[runtimeKey]).toBeUndefined(); + }); + expect(Array.isArray(definition.config_schema.required)).toBeTruthy(); + runtimeKeys.forEach((runtimeKey) => { + expect(definition.config_schema.required).not.toContain(runtimeKey); + }); const excerpt = connectorData.manager_contract_excerpt; expect(excerpt).toBeDefined(); @@ -1481,6 +1489,12 @@ describe('Connector Composer and Managed Connectors', () => { expect(result.data).toBeDefined(); expect(result.data?.managedConnectorEdit.name).toEqual('Updated IpInfo Connector'); expect(result.data?.managedConnectorEdit.manager_contract_configuration).toBeDefined(); + const runtimeKeys = ['OPENCTI_TOKEN', 'OPENCTI_URL', 'CONNECTOR_TYPE', 'CONNECTOR_RUN_AND_TERMINATE']; + runtimeKeys.forEach((runtimeKey) => { + const found = result.data?.managedConnectorEdit.manager_contract_configuration + .find((c: any) => c.key === runtimeKey); + expect(found).toBeUndefined(); + }); const autoConfig = result.data?.managedConnectorEdit.manager_contract_configuration .find((c: any) => c.key === 'CONNECTOR_AUTO'); expect(autoConfig.value).toEqual('false'); diff --git a/opencti-platform/opencti-graphql/tests/03-integration/02-resolvers/connector-migration-test.ts b/opencti-platform/opencti-graphql/tests/03-integration/02-resolvers/connector-migration-test.ts index 82fd584d4c96..c6e5bcb022c7 100644 --- a/opencti-platform/opencti-graphql/tests/03-integration/02-resolvers/connector-migration-test.ts +++ b/opencti-platform/opencti-graphql/tests/03-integration/02-resolvers/connector-migration-test.ts @@ -200,6 +200,10 @@ describe('Check connector migration', () => { rawConfig.filter((c: any) => !RUNTIME_KEYS.includes(c.key)); const actualConfig = rawConfig.filter((c: { key: string }) => !RUNTIME_KEYS.includes(c.key)); + RUNTIME_KEYS.forEach((runtimeKey) => { + const found = rawConfig.find((c: { key: string }) => c.key === runtimeKey); + expect(found).toBeUndefined(); + }); const schemaProperties = contractFound.config_schema.properties; const expectedKeys = Object.keys(schemaProperties); diff --git a/opencti-platform/opencti-graphql/tests/03-integration/10-modules/catalog/catalog-resolver-test.ts b/opencti-platform/opencti-graphql/tests/03-integration/10-modules/catalog/catalog-resolver-test.ts new file mode 100644 index 000000000000..e16523a1e021 --- /dev/null +++ b/opencti-platform/opencti-graphql/tests/03-integration/10-modules/catalog/catalog-resolver-test.ts @@ -0,0 +1,199 @@ +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import gql from 'graphql-tag'; +import { v4 as uuidv4 } from 'uuid'; +import { queryAsAdminWithSuccess } from '../../../utils/testQueryHelper'; +import { ADMIN_USER, testContext } from '../../../utils/testQuery'; +import { synchronizeCatalogs } from '../../../../src/modules/catalog/sync/catalog-sync-domain'; +import conf, { ENABLED_FEATURE_FLAGS, FEATURE_FLAG_ALL } from '../../../../src/config/conf'; + +const LIST_CATALOGS_QUERY = gql` + query Catalogs { + catalogs { + id + name + description + contracts + } + } +`; + +const GET_CATALOG_BY_ID_QUERY = gql` + query CatalogById($id: String!) { + catalog(id: $id) { + id + name + description + contracts + } + } +`; + +const GET_CONTRACT_BY_SLUG_QUERY = gql` + query ContractBySlug($slug: String!) { + contract(slug: $slug) { + catalog_id + contract + } + } +`; + +const assertContractShape = (contract: any) => { + expect(contract.title).toEqual(expect.any(String)); + expect(contract.slug).toEqual(expect.any(String)); + expect(contract.description).toEqual(expect.any(String)); + expect(contract.short_description).toEqual(expect.any(String)); + expect(contract.use_cases).toEqual(expect.any(Array)); + expect(contract.verified).toEqual(expect.any(Boolean)); + expect(contract.playbook_supported).toEqual(expect.any(Boolean)); + expect(contract.max_confidence_level).toEqual(expect.any(Number)); + expect(contract.source_code).toEqual(expect.any(String)); + expect(contract.manager_supported).toEqual(expect.any(Boolean)); + expect(contract.container_version).toEqual(expect.any(String)); + expect(contract.container_image).toEqual(expect.any(String)); + expect(contract.container_type).toEqual(expect.any(String)); + expect(contract.config_schema).toEqual(expect.any(Object)); + expect(contract.logo === null || typeof contract.logo === 'string').toBe(true); + expect(contract.last_verified_date === null || typeof contract.last_verified_date === 'string').toBe(true); + expect(contract.support_version === null || typeof contract.support_version === 'string').toBe(true); + expect(contract.subscription_link === null || typeof contract.subscription_link === 'string').toBe(true); + expect(contract.config_schema).toMatchObject({ + type: expect.any(String), + properties: expect.any(Object), + required: expect.any(Array), + additionalProperties: expect.any(Boolean), + }); +}; + +describe('Catalog resolver integration', () => { + const v0TemplatePath = fileURLToPath(new URL('./fixture/integration-catalog-v0.json', import.meta.url)); + const v1TemplatePath = fileURLToPath(new URL('./fixture/integration-catalog-v1.json', import.meta.url)); + let v0CatalogUri = ''; + let v1CatalogUri = ''; + let catalogV0Id = ''; + let catalogV1Id = ''; + let catalogV0Slug = ''; + let catalogV1Slug = ''; + let tempFixtureDir = ''; + const previousEnabledFeatureFlags = [...ENABLED_FEATURE_FLAGS]; + const previousCustomCatalogRefreshEndpoint = conf.get('catalog_manager:custom_catalog_refresh_endpoint_uri'); + const previousCustomCatalogs = conf.get('app:custom_catalogs'); + const runtimeKeys = ['OPENCTI_TOKEN', 'OPENCTI_URL', 'CONNECTOR_TYPE', 'CONNECTOR_RUN_AND_TERMINATE']; + + const assertCatalogEndpoints = async (expectedCatalogId: string, expectedSlug: string, customCatalogUri: string) => { + conf.set('catalog_manager:custom_catalog_refresh_endpoint_uri', customCatalogUri); + await synchronizeCatalogs(testContext, ADMIN_USER); + + const catalogsResult = await queryAsAdminWithSuccess({ + query: LIST_CATALOGS_QUERY, + variables: {}, + }); + const catalogs = catalogsResult.data?.catalogs ?? []; + const expectedCatalog = catalogs.find((catalog: any) => catalog.id === expectedCatalogId); + expect(expectedCatalog).toBeDefined(); + + for (const catalog of catalogs) { + expect(catalog).toMatchObject({ + id: expect.any(String), + name: expect.any(String), + description: expect.any(String), + contracts: expect.any(Array), + }); + const contracts = catalog.contracts.map((raw: string) => JSON.parse(raw)); + const slugs = contracts.map((contract: { slug: string }) => contract.slug); + expect(new Set(slugs).size).toEqual(slugs.length); + contracts.forEach((contract: any) => { + assertContractShape(contract); + runtimeKeys.forEach((runtimeKey) => { + expect(contract.config_schema.properties[runtimeKey]).toBeUndefined(); + }); + }); + } + + const byIdResult = await queryAsAdminWithSuccess({ + query: GET_CATALOG_BY_ID_QUERY, + variables: { id: expectedCatalogId }, + }); + const byIdCatalog = byIdResult.data?.catalog; + expect(byIdCatalog).toMatchObject({ + id: expectedCatalogId, + name: expect.any(String), + description: expect.any(String), + contracts: expect.any(Array), + }); + expect(byIdCatalog.contracts.length).toBeGreaterThan(0); + const byIdContract = JSON.parse(byIdCatalog.contracts[0]); + assertContractShape(byIdContract); + runtimeKeys.forEach((runtimeKey) => { + expect(byIdContract.config_schema.properties[runtimeKey]).toBeUndefined(); + }); + + const bySlugResult = await queryAsAdminWithSuccess({ + query: GET_CONTRACT_BY_SLUG_QUERY, + variables: { slug: expectedSlug }, + }); + const bySlugPayload = bySlugResult.data?.contract; + expect(bySlugPayload).toMatchObject({ + catalog_id: expectedCatalogId, + contract: expect.any(String), + }); + const bySlugContract = JSON.parse(bySlugPayload.contract); + assertContractShape(bySlugContract); + expect(bySlugContract.slug).toEqual(expectedSlug); + runtimeKeys.forEach((runtimeKey) => { + expect(bySlugContract.config_schema.properties[runtimeKey]).toBeUndefined(); + }); + }; + + beforeAll(async () => { + const runSuffix = uuidv4().slice(0, 8); + tempFixtureDir = await mkdtemp(join(tmpdir(), 'opencti-catalog-integration-')); + const v0Fixture = JSON.parse(await readFile(v0TemplatePath, 'utf8')); + const v1Fixture = JSON.parse(await readFile(v1TemplatePath, 'utf8')); + + catalogV0Id = `integration-catalog-v0-${runSuffix}`; + catalogV1Id = `integration-catalog-v1-${runSuffix}`; + catalogV0Slug = `integration-v0-connector-${runSuffix}`; + catalogV1Slug = `integration-v1-connector-${runSuffix}`; + + v0Fixture.id = catalogV0Id; + v0Fixture.contracts[0].slug = catalogV0Slug; + v1Fixture.id = catalogV1Id; + v1Fixture.contracts[0].slug = catalogV1Slug; + v1Fixture.contracts[0].id = `${catalogV1Slug}-${v1Fixture.contracts[0].version}`; + + const v0TempPath = join(tempFixtureDir, 'integration-catalog-v0.json'); + const v1TempPath = join(tempFixtureDir, 'integration-catalog-v1.json'); + await writeFile(v0TempPath, JSON.stringify(v0Fixture), 'utf8'); + await writeFile(v1TempPath, JSON.stringify(v1Fixture), 'utf8'); + v0CatalogUri = pathToFileURL(v0TempPath).toString(); + v1CatalogUri = pathToFileURL(v1TempPath).toString(); + + ENABLED_FEATURE_FLAGS.splice(0, ENABLED_FEATURE_FLAGS.length); + ENABLED_FEATURE_FLAGS.push(...previousEnabledFeatureFlags.filter((flag) => flag !== FEATURE_FLAG_ALL)); + if (!ENABLED_FEATURE_FLAGS.includes('DECOUPLING_VERSIONS')) { + ENABLED_FEATURE_FLAGS.push('DECOUPLING_VERSIONS'); + } + conf.set('app:custom_catalogs', []); + }); + + afterAll(async () => { + conf.set('catalog_manager:custom_catalog_refresh_endpoint_uri', previousCustomCatalogRefreshEndpoint); + conf.set('app:custom_catalogs', previousCustomCatalogs); + ENABLED_FEATURE_FLAGS.splice(0, ENABLED_FEATURE_FLAGS.length, ...previousEnabledFeatureFlags); + if (tempFixtureDir) { + await rm(tempFixtureDir, { recursive: true, force: true }); + } + }); + + it('should expose expected endpoint shapes with a custom V0 catalog source', async () => { + await assertCatalogEndpoints(catalogV0Id, catalogV0Slug, v0CatalogUri); + }); + + it('should expose expected endpoint shapes with a custom V1 catalog source', async () => { + await assertCatalogEndpoints(catalogV1Id, catalogV1Slug, v1CatalogUri); + }); +}); diff --git a/opencti-platform/opencti-graphql/tests/03-integration/10-modules/catalog/fixture/integration-catalog-v0.json b/opencti-platform/opencti-graphql/tests/03-integration/10-modules/catalog/fixture/integration-catalog-v0.json new file mode 100644 index 000000000000..33fce64bd08d --- /dev/null +++ b/opencti-platform/opencti-graphql/tests/03-integration/10-modules/catalog/fixture/integration-catalog-v0.json @@ -0,0 +1,50 @@ +{ + "id": "integration-catalog-v0", + "name": "Integration Catalog V0", + "description": "Small V0 catalog fixture for resolver integration tests", + "version": "7.2.0", + "contracts": [ + { + "title": "Integration V0 Connector", + "slug": "integration-v0-connector", + "description": "V0 contract fixture", + "short_description": "v0 fixture", + "logo": null, + "use_cases": [], + "verified": true, + "last_verified_date": "2026-01-01", + "playbook_supported": false, + "max_confidence_level": 50, + "support_version": ">= 7.0.0", + "subscription_link": null, + "source_code": "", + "manager_supported": true, + "container_version": "1.0.0", + "container_image": "opencti/connector-integration-v0", + "container_type": "EXTERNAL_IMPORT", + "config_schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "integration-v0", + "type": "object", + "properties": { + "INTEGRATION_API_KEY": { + "type": "string", + "description": "fixture key" + }, + "OPENCTI_URL": { + "type": "string", + "description": "runtime key to strip" + } + }, + "required": [ + "INTEGRATION_API_KEY", + "OPENCTI_URL" + ], + "additionalProperties": false + }, + "license_type": null, + "solution_categories": [], + "contact": null + } + ] +} diff --git a/opencti-platform/opencti-graphql/tests/03-integration/10-modules/catalog/fixture/integration-catalog-v1.json b/opencti-platform/opencti-graphql/tests/03-integration/10-modules/catalog/fixture/integration-catalog-v1.json new file mode 100644 index 000000000000..d7edcf0d2af3 --- /dev/null +++ b/opencti-platform/opencti-graphql/tests/03-integration/10-modules/catalog/fixture/integration-catalog-v1.json @@ -0,0 +1,54 @@ +{ + "id": "integration-catalog-v1", + "name": "Integration Catalog V1", + "description": "Small V1 catalog fixture for resolver integration tests", + "manifest_schema_version": "1", + "manifest_version": "2026.08.19", + "product_version": "7.2.0", + "contracts": [ + { + "id": "integration-v1-connector-1.0.0", + "title": "Integration V1 Connector", + "slug": "integration-v1-connector", + "description": "V1 contract fixture", + "short_description": "v1 fixture", + "logo": null, + "use_cases": [], + "verified": true, + "last_verified_date": "2026-01-01", + "subscription_link": null, + "source_code": null, + "manager_supported": true, + "support_version": "7.0.0", + "license_type": null, + "contact": null, + "solution_categories": [], + "version": "1.0.0", + "image_name": "opencti/connector-integration-v1", + "image_type": "EXTERNAL_IMPORT", + "additional_properties": { + "playbook_supported": false, + "max_confidence_level": 75 + }, + "config_schema": { + "type": "object", + "properties": { + "INTEGRATION_API_KEY": { + "type": "string", + "description": "fixture key" + }, + "OPENCTI_TOKEN": { + "type": "string", + "format": "password", + "description": "runtime key to strip" + } + }, + "required": [ + "INTEGRATION_API_KEY", + "OPENCTI_TOKEN" + ], + "additionalProperties": false + } + } + ] +} diff --git a/opencti-platform/opencti-graphql/tests/03-integration/10-modules/http/catalog-logo-http-test.ts b/opencti-platform/opencti-graphql/tests/03-integration/10-modules/http/catalog-logo-http-test.ts new file mode 100644 index 000000000000..565433862b40 --- /dev/null +++ b/opencti-platform/opencti-graphql/tests/03-integration/10-modules/http/catalog-logo-http-test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from 'vitest'; +import { API_URI, generateBasicAuth } from '../../../utils/testQuery'; +import { basePath } from '../../../../src/config/conf'; +import { rawUpload } from '../../../../src/database/raw-file-storage'; + +const CATALOG_LOGO_PATH = `${API_URI}${basePath}/catalog/logo`; + +describe('catalog logo HTTP integration', () => { + it('should return 403 when unauthenticated', async () => { + const res = await fetch(`${CATALOG_LOGO_PATH}/does-not-matter.png`); + expect(res.status).toBe(403); + }); + + it('should return 404 when authenticated but file does not exist', async () => { + const res = await fetch(`${CATALOG_LOGO_PATH}/missing-logo.png`, { + headers: { + authorization: generateBasicAuth(), + }, + }); + expect(res.status).toBe(404); + }); + + it('should return 200 and content-type when logo exists', async () => { + await rawUpload('catalog-logos/integration-logo.png', Buffer.from('png-content')); + const res = await fetch(`${CATALOG_LOGO_PATH}/integration-logo.png`, { + headers: { + authorization: generateBasicAuth(), + }, + }); + expect(res.status).toBe(200); + expect(res.headers.get('content-type')).toContain('image/png'); + }); +}); From 2b65e53bfc6f9041972d7883ec6000973839f140 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Wed, 19 Aug 2026 18:14:23 +0200 Subject: [PATCH 16/45] fix(manager): log when enterprise-only managers are skipped --- .../opencti-graphql/src/manager/managerModule.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/opencti-platform/opencti-graphql/src/manager/managerModule.ts b/opencti-platform/opencti-graphql/src/manager/managerModule.ts index ed84656d17cd..d990e064005f 100644 --- a/opencti-platform/opencti-graphql/src/manager/managerModule.ts +++ b/opencti-platform/opencti-graphql/src/manager/managerModule.ts @@ -64,6 +64,7 @@ const initManager = (manager: ManagerDefinition) => { let streamProcessor: StreamProcessor; let running = false; let shutdown = false; + let enterpriseBlockedLogged = false; const cronTimer = new InterruptibleTimer(); const streamTimer = new InterruptibleTimer(); @@ -71,8 +72,13 @@ const initManager = (manager: ManagerDefinition) => { const cronHandler = async (cronInputFn?: () => Promise) => { if (manager.cronSchedulerHandler) { if (!(await isEnterpriseEditionAuthorized(manager))) { + if (!enterpriseBlockedLogged) { + logApp.info(`[OPENCTI-MODULE] ${manager.label} not running (enterprise edition license required)`); + enterpriseBlockedLogged = true; + } return; } + enterpriseBlockedLogged = false; let lock; let cronInput; const startDate = utcDate(); @@ -114,8 +120,13 @@ const initManager = (manager: ManagerDefinition) => { const streamHandler = async () => { if (manager.streamSchedulerHandler) { if (!(await isEnterpriseEditionAuthorized(manager))) { + if (!enterpriseBlockedLogged) { + logApp.info(`[OPENCTI-MODULE] ${manager.label} not running (enterprise edition license required)`); + enterpriseBlockedLogged = true; + } return; } + enterpriseBlockedLogged = false; let lock; try { // Lock the manager From 67fe0cd353de3bde691ca5810c126fa0895b16df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Wed, 19 Aug 2026 18:14:26 +0200 Subject: [PATCH 17/45] refactor(catalog): tune manager gating and sync log levels --- .../src/modules/catalog/catalog-manager.ts | 2 +- .../src/modules/catalog/sync/catalog-sync-domain.ts | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-manager.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-manager.ts index 73debdd0e80d..5b6b4c3a0a18 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-manager.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-manager.ts @@ -34,7 +34,7 @@ const CATALOG_MANAGER_DEFINITION: ManagerDefinition = { enabledToStart(): boolean { return this.enabledByConfig && !!CATALOG_MANAGER_LOCK_KEY; }, - enterpriseEditionOnly: true, + enterpriseEditionOnly: false, cronSchedulerHandler: { handler: catalogManagerHandler, interval: CATALOG_MANAGER_INTERVAL, diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts b/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts index 1b7d5fde3c44..aa853f719fe1 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts @@ -266,7 +266,8 @@ const synchronizeCatalog = async (context: AuthContext, user: AuthUser, sourceCo module: 'catalog', }); } else { - logApp.debug('[OPENCTI-MODULE] New catalog source', { + logApp.info('[OPENCTI-MODULE] New catalog source', { + executionContext: context.source, sourceKind: sourceConfig.kind, module: 'catalog', }); @@ -314,7 +315,12 @@ const synchronizeCatalog = async (context: AuthContext, user: AuthUser, sourceCo }); // Persist refreshed catalog, contracts & logos // Something's fishy if this occurs too frequently. +<<<<<<< HEAD logApp.warn('[OPENCTI-MODULE] Persisting catalog & contracts', { +======= + logCatalog.debug('[OPENCTI-MODULE] Persisting catalog & contracts', { + executionContext: context.source, +>>>>>>> bf47229a2b (refactor(catalog): tune manager gating and sync log levels (#0)) catalogId: sourceCatalog.id, contractsCreationsCount: catalogSyncDiff.contractsCreations.length, contractsUpdatesCount: catalogSyncDiff.contractsUpdates.length, @@ -429,8 +435,12 @@ const cleanupObsoleteCatalogs = async (context: AuthContext, syncedCatalogs: str export const synchronizeCatalogs = async (context: AuthContext, user: AuthUser) => { const sources = initSyncSources(); const filigranCatalogRemoteUri = getFiligranCatalogRemoteUri(); +<<<<<<< HEAD logApp.debug('[OPENCTI-MODULE] Synchronizing catalogs', { module: 'catalog', +======= + logCatalog.info('[OPENCTI-MODULE] Synchronizing catalogs', { +>>>>>>> bf47229a2b (refactor(catalog): tune manager gating and sync log levels (#0)) executionContext: context.source, count: sources.length, filigranCatalogRemoteUri, From 71c2e497db01b3ce8d9c0a79beb739ad20ae75aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Wed, 19 Aug 2026 18:21:20 +0200 Subject: [PATCH 18/45] fix: translations --- .../opencti-front/lang/back/de.json | 29 ++++++++++++++++++- .../opencti-front/lang/back/en.json | 29 ++++++++++++++++++- .../opencti-front/lang/back/es.json | 29 ++++++++++++++++++- .../opencti-front/lang/back/fr.json | 29 ++++++++++++++++++- .../opencti-front/lang/back/it.json | 29 ++++++++++++++++++- .../opencti-front/lang/back/ja.json | 29 ++++++++++++++++++- .../opencti-front/lang/back/ko.json | 29 ++++++++++++++++++- .../opencti-front/lang/back/ru.json | 29 ++++++++++++++++++- .../opencti-front/lang/back/zh.json | 29 ++++++++++++++++++- 9 files changed, 252 insertions(+), 9 deletions(-) diff --git a/opencti-platform/opencti-front/lang/back/de.json b/opencti-platform/opencti-front/lang/back/de.json index ecb9440ea1b5..9bcbe3d0152a 100644 --- a/opencti-platform/opencti-front/lang/back/de.json +++ b/opencti-platform/opencti-front/lang/back/de.json @@ -108,6 +108,10 @@ "Card number": "Kartennummer", "Carousel order": "Karussell-Reihenfolge", "Case template": "Fallvorlage", + "Catalog Contract content hash": "Hash des Inhalts des Katalogvertrags", + "Catalog Contract ID": "ID des Katalogvertrags", + "Catalog ID": "Katalog-ID", + "Catalog product version": "Produktversion des Katalogs", "Category": "Kategorie", "CC": "CC", "Certifiate policies": "Zertifizierungsrichtlinien", @@ -130,11 +134,13 @@ "Confidence level": "Konfidenzniveau", "Confidentiality impact": "Auswirkungen auf die Vertraulichkeit", "Configuration": "Konfiguration", + "Configuration schema": "Konfigurationsschema", "Configuration version": "Version der Konfiguration", "Connector catalog": "Stecker-Katalog", "Connector ID": "Anschluss-ID", "Connector information": "Informationen zum Anschluss", "Connector manager": "Stecker-Manager", + "Connector manager contract": "Vertrag des Connector-Managers", "Connector manager current status": "Aktueller Status des Verbindungsmanagers", "Connector manager image": "Bild des Verbindungsmanagers", "Connector manager requested status": "Verbindungsmanager angeforderter Status", @@ -143,10 +149,12 @@ "Connector state": "Zustand des Verbinders", "Connector trigger filters": "Anschluss Triggerfilter", "Connector type": "Steckertyp", + "Connector upgrade strategy": "Upgrade-Strategie für den Connector", "Connector user id": "Connector Benutzer-ID", "Connectors": "Verbinder", "Consent confirm text": "Text zur Bestätigung der Zustimmung", "Consent message": "Zustimmungsnachricht", + "Contact": "Kontakt", "Contact information": "Kontaktinformationen", "Contains": "Enthält", "Contains observable": "Enthält beobachtbar", @@ -160,6 +168,7 @@ "Context": "Kontext", "Context data": "Kontextdaten", "Contextual only": "Nur kontextabhängig", + "Contract version": "Vertragsversion", "Copy confidence level to OpenCTI scores for indicators": "Kopieren des Konfidenzniveaus in OpenCTI-Bewertungen für Indikatoren", "Copy OpenCTI scores to confidence level for indicators": "Kopieren der OpenCTI-Ergebnisse in das Konfidenzniveau der Indikatoren", "Count": "Zählung", @@ -562,12 +571,14 @@ "Last score evolution": "Entwicklung des letzten Punktestandes", "Last seen": "Zuletzt gesehen", "Last update": "Letzte Aktualisierung", + "Last verified date": "Datum der letzten Überprüfung", "Lastname": "Nachname", "Latest containers": "Letzte Container", "Latest created relationships": "Zuletzt erstellte Beziehungen", "Latitude": "Breitengrad", "Layers": "Ebenen", "LdapStrategy": "LDAP-Strategie", + "License type": "Lizenztyp", "Lifetime": "Lebenszeit", "Light accent": "Hell akzentuiert", "Light background": "Heller Hintergrund", @@ -592,6 +603,7 @@ "Locations propagation in reports": "Ortsausbreitung in Berichten", "Log sources": "Protokollquellen", "Login message": "Login-Nachricht", + "Logo URI in file storage": "Logo-URI im Dateispeicher", "Logout remote": "Fernabmeldung", "Longitude": "Längengrad", "LZJD": "LZJD", @@ -607,8 +619,11 @@ "Manager contracts": "Manager-Verträge", "Manager ID": "Manager-ID", "Manager running": "Manager läuft", + "Manager supported": "Unterstützter Manager", "Mandatory": "Erforderlich", "Manifest": "Manifest", + "Manifest schema version": "Manifest-Schema-Version", + "Manifest version": "Manifest-Version", "Mappings": "Mappings", "Marital status": "Familienstand", "Marked for deletion at": "Zur Löschung vorgemerkt unter", @@ -626,6 +641,7 @@ "Match Pir with source of relationship": "Pirat mit Quelle der Beziehung abgleichen", "Max concurrent sessions (0 equals no maximum)": "Max. gleichzeitige Sitzungen (0 entspricht keinem Maximum)", "Max Confidence": "Maximale Konfidenz", + "Max confidence level": "Maximales Konfidenzniveau", "Max policy length": "Max policy length", "Max shareable markings": "Maximal teilbare Markierungen", "Max value": "Maximalwert", @@ -807,6 +823,7 @@ "Platforms": "Plattformen", "Playbook definition": "Spielbuch-Definition", "Playbook running": "Spielbuch läuft", + "Playbook supported": "Unterstütztes Playbook", "Policy constraints": "Policy-Beschränkungen", "Policy mappings": "Richtlinien-Zuordnungen", "Port": "Port", @@ -914,6 +931,7 @@ "Result": "Ergebnis", "Result name": "Name des Ergebnisses", "Retention manager": "Leiter der Personalabteilung", + "Revision": "Revision", "Revoke score": "Widerrufener Punktestand", "Revoked": "Widerrufen", "RIR": "RIR", @@ -959,6 +977,7 @@ "SHA3-256": "SHA3-256", "SHA3-512": "SHA3-512", "Shared with": "Freigegeben für", + "Short description": "Kurzbeschreibung", "Show submenu icons": "Symbole für Untermenüs anzeigen", "Sightings of observables via observed data": "Sichtungen von Observablen über beobachtete Daten", "Sightings propagation from indicator": "Fortpflanzung von Beobachtungen über einen Indikator", @@ -970,12 +989,15 @@ "SMTP configuration": "SMTP-Konfiguration", "SMTP enabled": "SMTP aktiviert", "Software Vulnerability propagation to Infrastructure": "Ausbreitung von Software-Sicherheitslücken auf die Infrastruktur", + "Solution categories": "Lösungskategorien", "Sophistication": "Raffinesse", "Source": "Quelle", + "Source code": "Quellcode", "Source connector": "Quellverbinder", "Source entity": "Quelle Entität", "Source name": "Name der Quelle", "Source type": "Quellentyp", + "Source URI": "Quell-URI", "SRC": "SRC", "SRC byte count": "SRC-Bytezahl", "SRC packets": "SRC-Pakete", @@ -1015,6 +1037,8 @@ "Subject public key exponent": "Thema Exponent des öffentlichen Schlüssels", "Subject public key modulus": "Betreff öffentlicher Schlüsselmodul", "Submission date": "Datum der Einreichung", + "Subscription link": "Abonnement-Link", + "Support version": "Unterstützte Version", "SWID": "SWID", "Sync workflow statuses by name": "Workflow-Status nach Namen synchronisieren", "Synchronized": "Synchronisiert", @@ -1104,6 +1128,7 @@ "URL": "URL", "Usage propagation of parent techniques": "Verwendungspropagierung von übergeordneten Techniken", "Usage propagation via attribution": "Weitergabe der Nutzung über Attribution", + "Use cases": "Anwendungsfälle", "Use DB configuration": "DB-Konfiguration verwenden", "Use SSL": "SSL verwenden", "User": "Benutzer", @@ -1136,6 +1161,7 @@ "Variables": "Variablen", "Vendor": "Anbieter", "verb": "verb", + "Verified": "Verifiziert", "Verify SSL certificate": "SSL-Zertifikat verifizieren", "Version": "Version", "Version History": "Versionsgeschichte", @@ -1181,4 +1207,5 @@ "XTM One intent": "XTM Eine Absicht", "XTM One registration manager": "XTM One Registrierungsmanager", "XTM1 CGU acceptance status": "XTM1 CGU-Annahmestatus" -} \ No newline at end of file +} + diff --git a/opencti-platform/opencti-front/lang/back/en.json b/opencti-platform/opencti-front/lang/back/en.json index f9d39eda4774..cb33c4af0384 100644 --- a/opencti-platform/opencti-front/lang/back/en.json +++ b/opencti-platform/opencti-front/lang/back/en.json @@ -108,6 +108,10 @@ "Card number": "Card number", "Carousel order": "Carousel order", "Case template": "Case template", + "Catalog Contract content hash": "Catalog Contract content hash", + "Catalog Contract ID": "Catalog Contract ID", + "Catalog ID": "Catalog ID", + "Catalog product version": "Catalog product version", "Category": "Category", "CC": "CC", "Certifiate policies": "Certifiate policies", @@ -130,11 +134,13 @@ "Confidence level": "Confidence level", "Confidentiality impact": "Confidentiality impact", "Configuration": "Configuration", + "Configuration schema": "Configuration schema", "Configuration version": "Configuration version", "Connector catalog": "Connector catalog", "Connector ID": "Connector ID", "Connector information": "Connector information", "Connector manager": "Connector manager", + "Connector manager contract": "Connector manager contract", "Connector manager current status": "Connector manager current status", "Connector manager image": "Connector manager image", "Connector manager requested status": "Connector manager requested status", @@ -143,10 +149,12 @@ "Connector state": "Connector state", "Connector trigger filters": "Connector trigger filters", "Connector type": "Connector type", + "Connector upgrade strategy": "Connector upgrade strategy", "Connector user id": "Connector user id", "Connectors": "Connectors", "Consent confirm text": "Consent confirm text", "Consent message": "Consent message", + "Contact": "Contact", "Contact information": "Contact information", "Contains": "Contains", "Contains observable": "Contains observable", @@ -160,6 +168,7 @@ "Context": "Context", "Context data": "Context data", "Contextual only": "Contextual only", + "Contract version": "Contract version", "Copy confidence level to OpenCTI scores for indicators": "Copy confidence level to OpenCTI scores for indicators", "Copy OpenCTI scores to confidence level for indicators": "Copy OpenCTI scores to confidence level for indicators", "Count": "Count", @@ -562,12 +571,14 @@ "Last score evolution": "Last score evolution", "Last seen": "Last seen", "Last update": "Last update", + "Last verified date": "Last verified date", "Lastname": "Lastname", "Latest containers": "Latest containers", "Latest created relationships": "Latest created relationships", "Latitude": "Latitude", "Layers": "Layers", "LdapStrategy": "LDAP Strategy", + "License type": "License type", "Lifetime": "Lifetime", "Light accent": "Light accent", "Light background": "Light background", @@ -592,6 +603,7 @@ "Locations propagation in reports": "Locations propagation in reports", "Log sources": "Log sources", "Login message": "Login message", + "Logo URI in file storage": "Logo URI in file storage", "Logout remote": "Logout remote", "Longitude": "Longitude", "LZJD": "LZJD", @@ -607,8 +619,11 @@ "Manager contracts": "Manager contracts", "Manager ID": "Manager ID", "Manager running": "Manager running", + "Manager supported": "Manager supported", "Mandatory": "Mandatory", "Manifest": "Manifest", + "Manifest schema version": "Manifest schema version", + "Manifest version": "Manifest version", "Mappings": "Mappings", "Marital status": "Marital status", "Marked for deletion at": "Marked for deletion at", @@ -626,6 +641,7 @@ "Match Pir with source of relationship": "Match Pir with source of relationship", "Max concurrent sessions (0 equals no maximum)": "Max concurrent sessions (0 equals no maximum)", "Max Confidence": "Max Confidence", + "Max confidence level": "Max confidence level", "Max policy length": "Max policy length", "Max shareable markings": "Max shareable markings", "Max value": "Max value", @@ -807,6 +823,7 @@ "Platforms": "Platforms", "Playbook definition": "Playbook definition", "Playbook running": "Playbook running", + "Playbook supported": "Playbook supported", "Policy constraints": "Policy constraints", "Policy mappings": "Policy mappings", "Port": "Port", @@ -914,6 +931,7 @@ "Result": "Result", "Result name": "Result name", "Retention manager": "Retention manager", + "Revision": "Revision", "Revoke score": "Revoke score", "Revoked": "Revoked", "RIR": "RIR", @@ -959,6 +977,7 @@ "SHA3-256": "SHA3-256", "SHA3-512": "SHA3-512", "Shared with": "Shared with", + "Short description": "Short description", "Show submenu icons": "Show submenu icons", "Sightings of observables via observed data": "Sightings of observables via observed data", "Sightings propagation from indicator": "Sightings propagation from indicator", @@ -970,12 +989,15 @@ "SMTP configuration": "SMTP configuration", "SMTP enabled": "SMTP enabled", "Software Vulnerability propagation to Infrastructure": "Software Vulnerability propagation to Infrastructure", + "Solution categories": "Solution categories", "Sophistication": "Sophistication", "Source": "Source", + "Source code": "Source code", "Source connector": "Source connector", "Source entity": "Source entity", "Source name": "Source name", "Source type": "Source type", + "Source URI": "Source URI", "SRC": "SRC", "SRC byte count": "SRC byte count", "SRC packets": "SRC packets", @@ -1015,6 +1037,8 @@ "Subject public key exponent": "Subject public key exponent", "Subject public key modulus": "Subject public key modulus", "Submission date": "Submission date", + "Subscription link": "Subscription link", + "Support version": "Support version", "SWID": "SWID", "Sync workflow statuses by name": "Sync workflow statuses by name", "Synchronized": "Synchronized", @@ -1104,6 +1128,7 @@ "URL": "URL", "Usage propagation of parent techniques": "Usage propagation of parent techniques", "Usage propagation via attribution": "Usage propagation via attribution", + "Use cases": "Use cases", "Use DB configuration": "Use DB configuration", "Use SSL": "Use SSL", "User": "User", @@ -1136,6 +1161,7 @@ "Variables": "Variables", "Vendor": "Vendor", "verb": "verb", + "Verified": "Verified", "Verify SSL certificate": "Verify SSL certificate", "Version": "Version", "Version History": "Version History", @@ -1181,4 +1207,5 @@ "XTM One intent": "XTM One intent", "XTM One registration manager": "XTM One registration manager", "XTM1 CGU acceptance status": "XTM1 CGU acceptance status" -} \ No newline at end of file +} + diff --git a/opencti-platform/opencti-front/lang/back/es.json b/opencti-platform/opencti-front/lang/back/es.json index 333202b9d697..cb5f9c042208 100644 --- a/opencti-platform/opencti-front/lang/back/es.json +++ b/opencti-platform/opencti-front/lang/back/es.json @@ -108,6 +108,10 @@ "Card number": "Número de tarjeta", "Carousel order": "Orden del carrusel", "Case template": "Plantilla de casos", + "Catalog Contract content hash": "Hash del contenido del contrato del catálogo", + "Catalog Contract ID": "ID del contrato del catálogo", + "Catalog ID": "ID del catálogo", + "Catalog product version": "Versión del producto del catálogo", "Category": "Categoría", "CC": "CC", "Certifiate policies": "Políticas de certificación", @@ -130,11 +134,13 @@ "Confidence level": "Nivel de confianza", "Confidentiality impact": "Impacto en la confidencialidad", "Configuration": "Configuración", + "Configuration schema": "Esquema de configuración", "Configuration version": "Versión de configuración", "Connector catalog": "Catálogo de conectores", "Connector ID": "ID del conector", "Connector information": "Información sobre el conector", "Connector manager": "Gestor de conectores", + "Connector manager contract": "Contrato del gestor de conectores", "Connector manager current status": "Estado actual del gestor de conectores", "Connector manager image": "Imagen del gestor de conectores", "Connector manager requested status": "Estado solicitado del gestor de conectores", @@ -143,10 +149,12 @@ "Connector state": "Estado del conector", "Connector trigger filters": "Filtros de activación de conectores", "Connector type": "Tipo de conector", + "Connector upgrade strategy": "Estrategia de actualización de conectores", "Connector user id": "ID de usuario del conector", "Connectors": "Conectores", "Consent confirm text": "Texto de confirmación del consentimiento", "Consent message": "Mensaje de consentimiento", + "Contact": "Contacto", "Contact information": "Información de contacto", "Contains": "Contiene", "Contains observable": "Contiene observable", @@ -160,6 +168,7 @@ "Context": "Contexto", "Context data": "Datos contextuales", "Contextual only": "Sólo contextual", + "Contract version": "Versión del contrato", "Copy confidence level to OpenCTI scores for indicators": "Copiar el nivel de confianza a las puntuaciones OpenCTI para los indicadores", "Copy OpenCTI scores to confidence level for indicators": "Copiar las puntuaciones de OpenCTI en el nivel de confianza de los indicadores", "Count": "Cuenta", @@ -562,12 +571,14 @@ "Last score evolution": "Evolución de la última puntuación", "Last seen": "Visto por última vez", "Last update": "Última actualización", + "Last verified date": "Fecha de la última verificación", "Lastname": "Apellido", "Latest containers": "Últimos contenedores", "Latest created relationships": "Últimas relaciones creadas", "Latitude": "Latitud", "Layers": "Capas", "LdapStrategy": "Estrategia LDAP", + "License type": "Tipo de licencia", "Lifetime": "De por vida", "Light accent": "Acento claro", "Light background": "Fondo claro", @@ -592,6 +603,7 @@ "Locations propagation in reports": "Propagación de localizaciones en informes", "Log sources": "Fuentes de registro", "Login message": "Mensaje de inicio de sesión", + "Logo URI in file storage": "URI del logotipo en el almacenamiento de archivos", "Logout remote": "Cerrar sesión remota", "Longitude": "Longitud", "LZJD": "LZJD", @@ -607,8 +619,11 @@ "Manager contracts": "Gestor contratos", "Manager ID": "ID Gestor", "Manager running": "Gestor en funcionamiento", + "Manager supported": "Gestor compatible", "Mandatory": "Obligatorio", "Manifest": "Manifiesto", + "Manifest schema version": "Versión del esquema del manifiesto", + "Manifest version": "Versión del manifiesto", "Mappings": "Asignaciones", "Marital status": "Estado civil", "Marked for deletion at": "Marcado para eliminación en", @@ -626,6 +641,7 @@ "Match Pir with source of relationship": "Emparejar Pir con la fuente de relación", "Max concurrent sessions (0 equals no maximum)": "Máximo de sesiones simultáneas (0 equivale a ningún máximo)", "Max Confidence": "Confianza máxima", + "Max confidence level": "Nivel máximo de confianza", "Max policy length": "Longitud máxima de la política", "Max shareable markings": "Marcas máximas compartibles", "Max value": "Valor máximo", @@ -807,6 +823,7 @@ "Platforms": "Plataformas", "Playbook definition": "Definición de Playbook", "Playbook running": "Ejecución de Playbook", + "Playbook supported": "Guía de procedimientos compatible", "Policy constraints": "Restricciones políticas", "Policy mappings": "Asignación de políticas", "Port": "Puerto", @@ -914,6 +931,7 @@ "Result": "Resultado", "Result name": "Nombre del resultado", "Retention manager": "Responsable de retención", + "Revision": "Revisión", "Revoke score": "Puntuación de revocación", "Revoked": "Revocado", "RIR": "RIR", @@ -959,6 +977,7 @@ "SHA3-256": "SHA3-256", "SHA3-512": "SHA3-512", "Shared with": "Compartido con", + "Short description": "Breve descripción", "Show submenu icons": "Mostrar iconos de submenú", "Sightings of observables via observed data": "Avistamientos de observables a través de datos observados", "Sightings propagation from indicator": "Propagación de avistamientos a partir de un indicador", @@ -970,12 +989,15 @@ "SMTP configuration": "Configuración SMTP", "SMTP enabled": "SMTP habilitado", "Software Vulnerability propagation to Infrastructure": "Propagación de vulnerabilidades de software a la infraestructura", + "Solution categories": "Categorías de la solución", "Sophistication": "Sofisticación", "Source": "Fuente", + "Source code": "Código fuente", "Source connector": "Conector fuente", "Source entity": "Entidad de origen", "Source name": "Nombre de la fuente", "Source type": "Tipo de fuente", + "Source URI": "URI de origen", "SRC": "SRC", "SRC byte count": "Recuento de bytes SRC", "SRC packets": "Paquetes SRC", @@ -1015,6 +1037,8 @@ "Subject public key exponent": "Tema exponente de clave pública", "Subject public key modulus": "Sujeto módulo de clave pública", "Submission date": "Fecha de envío", + "Subscription link": "Enlace de suscripción", + "Support version": "Versión compatible", "SWID": "SWID", "Sync workflow statuses by name": "Sincronizar los estados del flujo de trabajo por nombre", "Synchronized": "Sincronizado", @@ -1104,6 +1128,7 @@ "URL": "URL", "Usage propagation of parent techniques": "Propagación de uso de técnicas parentales", "Usage propagation via attribution": "Propagación del uso mediante atribución", + "Use cases": "Casos de uso", "Use DB configuration": "Usar configuración de la base de datos", "Use SSL": "Usar SSL", "User": "Usuario", @@ -1136,6 +1161,7 @@ "Variables": "Variables", "Vendor": "Proveedor", "verb": "verbo", + "Verified": "Verificado", "Verify SSL certificate": "Verificar certificado SSL", "Version": "Versión", "Version History": "Historial de versiones", @@ -1181,4 +1207,5 @@ "XTM One intent": "Intento de XTM One", "XTM One registration manager": "Gestor de inscripciones XTM One", "XTM1 CGU acceptance status": "XTM1 Estado de aceptación de la UGC" -} \ No newline at end of file +} + diff --git a/opencti-platform/opencti-front/lang/back/fr.json b/opencti-platform/opencti-front/lang/back/fr.json index e8f2339a5074..a703b821655a 100644 --- a/opencti-platform/opencti-front/lang/back/fr.json +++ b/opencti-platform/opencti-front/lang/back/fr.json @@ -108,6 +108,10 @@ "Card number": "Numéro de carte", "Carousel order": "Ordre du carrousel", "Case template": "Modèle de cas", + "Catalog Contract content hash": "Hachage du contenu du contrat de catalogue", + "Catalog Contract ID": "ID du contrat de catalogue", + "Catalog ID": "ID du catalogue", + "Catalog product version": "Version du produit du catalogue", "Category": "Catégorie", "CC": "CC", "Certifiate policies": "Politiques de certification", @@ -130,11 +134,13 @@ "Confidence level": "Niveau de confiance", "Confidentiality impact": "Impact sur la confidentialité", "Configuration": "Configuration", + "Configuration schema": "Schéma de configuration", "Configuration version": "Version de la configuration", "Connector catalog": "Catalogue des connecteurs", "Connector ID": "ID du connecteur", "Connector information": "Informations sur le connecteur", "Connector manager": "Gestionnaire du connecteur", + "Connector manager contract": "Contrat du gestionnaire de connecteurs", "Connector manager current status": "Statut actuel du gestionnaire de connecteur", "Connector manager image": "Image du gestionnaire de connecteur", "Connector manager requested status": "Statut demandé par le gestionnaire de connecteur", @@ -143,10 +149,12 @@ "Connector state": "État du connecteur", "Connector trigger filters": "Filtres de déclenchement du connecteur", "Connector type": "Type de connecteur", + "Connector upgrade strategy": "Stratégie de mise à niveau des connecteurs", "Connector user id": "Identifiant de l'utilisateur du connecteur", "Connectors": "Connecteurs", "Consent confirm text": "Texte de confirmation du consentement", "Consent message": "Message de consentement", + "Contact": "Contact", "Contact information": "Informations de contact", "Contains": "Contient", "Contains observable": "Contient des observables", @@ -160,6 +168,7 @@ "Context": "Contexte", "Context data": "Données contextuelles", "Contextual only": "Contextuel uniquement", + "Contract version": "Version du contrat", "Copy confidence level to OpenCTI scores for indicators": "Copier le niveau de confiance dans les scores OpenCTI pour les indicateurs", "Copy OpenCTI scores to confidence level for indicators": "Copier les scores OpenCTI vers le niveau de confiance pour les indicateurs", "Count": "Nombre", @@ -562,12 +571,14 @@ "Last score evolution": "Evolution du dernier score", "Last seen": "Dernière fois qu'il a été vu", "Last update": "Dernière mise à jour", + "Last verified date": "Date de la dernière vérification", "Lastname": "Nom", "Latest containers": "Derniers conteneurs", "Latest created relationships": "Dernières relations créées", "Latitude": "Latitude", "Layers": "Couches", "LdapStrategy": "Stratégie LDAP", + "License type": "Type de licence", "Lifetime": "Durée de vie", "Light accent": "Accentuation de la lumière", "Light background": "Fond clair", @@ -592,6 +603,7 @@ "Locations propagation in reports": "Propagation de l'emplacement dans les rapports", "Log sources": "Sources de logs", "Login message": "Message de connexion", + "Logo URI in file storage": "URI du logo dans le stockage de fichiers", "Logout remote": "Déconnexion à distance", "Longitude": "Longitude", "LZJD": "LZJD", @@ -607,8 +619,11 @@ "Manager contracts": "Contrats du gestionnaire", "Manager ID": "Manager ID", "Manager running": "Gestionnaire en cours d'exécution", + "Manager supported": "Gestionnaire pris en charge", "Mandatory": "Obligatoire", "Manifest": "Manifeste", + "Manifest schema version": "Version du schéma du manifeste", + "Manifest version": "Version du manifeste", "Mappings": "Mappages", "Marital status": "État civil", "Marked for deletion at": "Marqué pour suppression à", @@ -626,6 +641,7 @@ "Match Pir with source of relationship": "Faire correspondre Pir à la source de la relation", "Max concurrent sessions (0 equals no maximum)": "Nombre maximal de sessions simultanées (0 signifie qu'il n'y a pas de maximum)", "Max Confidence": "Confiance maximale", + "Max confidence level": "Niveau de confiance maximal", "Max policy length": "Longueur maximale de la politique", "Max shareable markings": "Marques maximales partageables", "Max value": "Valeur maximale", @@ -807,6 +823,7 @@ "Platforms": "Plateformes", "Playbook definition": "Définition du Playbook", "Playbook running": "Playbook en cours d'exécution", + "Playbook supported": "Guide de procédures pris en charge", "Policy constraints": "Contraintes de politique", "Policy mappings": "Mappages de politiques", "Port": "Port", @@ -914,6 +931,7 @@ "Result": "Résultat", "Result name": "Nom du résultat", "Retention manager": "Responsable de la rétention", + "Revision": "Révision", "Revoke score": "Score de révocation", "Revoked": "Révoqué", "RIR": "RIR", @@ -959,6 +977,7 @@ "SHA3-256": "SHA3-256", "SHA3-512": "SHA3-512", "Shared with": "Partagé avec", + "Short description": "Brève description", "Show submenu icons": "Afficher les icônes des sous-menus", "Sightings of observables via observed data": "Observations d'observables via des données observées", "Sightings propagation from indicator": "Propagation des observations à partir d'un indicateur", @@ -970,12 +989,15 @@ "SMTP configuration": "Configuration SMTP", "SMTP enabled": "SMTP activé", "Software Vulnerability propagation to Infrastructure": "Propagation d'une vulnérabilité d'un logiciel à une infrastructure", + "Solution categories": "Catégories de solutions", "Sophistication": "Sophistication", "Source": "Source", + "Source code": "Code source", "Source connector": "Connecteur source", "Source entity": "Entité source", "Source name": "Nom de la source", "Source type": "Type de source", + "Source URI": "URI de la source", "SRC": "SRC", "SRC byte count": "Nombre d'octets SRC", "SRC packets": "Paquets SRC", @@ -1015,6 +1037,8 @@ "Subject public key exponent": "Sujet clé publique exposant", "Subject public key modulus": "Sujet modulus de clé publique", "Submission date": "Date de soumission", + "Subscription link": "Lien d’abonnement", + "Support version": "Version prise en charge", "SWID": "SWID", "Sync workflow statuses by name": "Synchroniser les statuts des flux de travail par nom", "Synchronized": "Synchronisé", @@ -1104,6 +1128,7 @@ "URL": "URL", "Usage propagation of parent techniques": "Propagation de l'usage des techniques parentales", "Usage propagation via attribution": "Propagation de l'usage par attribution", + "Use cases": "Cas d’utilisation", "Use DB configuration": "Utiliser la configuration en base", "Use SSL": "Utiliser SSL", "User": "Utilisateur", @@ -1136,6 +1161,7 @@ "Variables": "Variables", "Vendor": "Vendeur", "verb": "verbe", + "Verified": "Vérifié", "Verify SSL certificate": "Vérifier le certificat SSL", "Version": "Version", "Version History": "Historique de la version", @@ -1181,4 +1207,5 @@ "XTM One intent": "XTM One intent", "XTM One registration manager": "Gestionnaire d'enregistrement XTM One", "XTM1 CGU acceptance status": "Statut d'acceptation des CGU XTM1" -} \ No newline at end of file +} + diff --git a/opencti-platform/opencti-front/lang/back/it.json b/opencti-platform/opencti-front/lang/back/it.json index 1a0b8e3acd8e..fc52545ab595 100644 --- a/opencti-platform/opencti-front/lang/back/it.json +++ b/opencti-platform/opencti-front/lang/back/it.json @@ -108,6 +108,10 @@ "Card number": "Numero di carta", "Carousel order": "Ordine del carosello", "Case template": "Modello di caso", + "Catalog Contract content hash": "Hash del contenuto del contratto del catalogo", + "Catalog Contract ID": "ID del contratto del catalogo", + "Catalog ID": "ID del catalogo", + "Catalog product version": "Versione del prodotto del catalogo", "Category": "Categoria", "CC": "CC", "Certifiate policies": "Politiche del certificato", @@ -130,11 +134,13 @@ "Confidence level": "Livello di fiducia", "Confidentiality impact": "Impatto sulla riservatezza", "Configuration": "Configurazione", + "Configuration schema": "Schema di configurazione", "Configuration version": "Versione della configurazione", "Connector catalog": "Catalogo dei connettori", "Connector ID": "ID connettore", "Connector information": "Informazioni sul connettore", "Connector manager": "Gestore del connettore", + "Connector manager contract": "Contratto del gestore dei connettori", "Connector manager current status": "Stato attuale del gestore di connettori", "Connector manager image": "Immagine del gestore del connettore", "Connector manager requested status": "Stato richiesto dal gestore del connettore", @@ -143,10 +149,12 @@ "Connector state": "Stato del connettore", "Connector trigger filters": "Filtri di attivazione del connettore", "Connector type": "Tipo di connettore", + "Connector upgrade strategy": "Strategia di aggiornamento dei connettori", "Connector user id": "ID utente del connettore", "Connectors": "Connettori", "Consent confirm text": "Testo di conferma del consenso", "Consent message": "Messaggio di consenso", + "Contact": "Contatti", "Contact information": "Informazioni di contatto", "Contains": "Contiene", "Contains observable": "Contiene osservabile", @@ -160,6 +168,7 @@ "Context": "Contesto", "Context data": "Dati di contesto", "Contextual only": "Solo contestuale", + "Contract version": "Versione del contratto", "Copy confidence level to OpenCTI scores for indicators": "Copia il livello di fiducia nei punteggi OpenCTI per gli indicatori", "Copy OpenCTI scores to confidence level for indicators": "Copia i punteggi OpenCTI nel livello di fiducia per gli indicatori", "Count": "Conteggio", @@ -562,12 +571,14 @@ "Last score evolution": "Evoluzione dell'ultimo punteggio", "Last seen": "Visto per l'ultima volta", "Last update": "Ultimo aggiornamento", + "Last verified date": "Data dell’ultima verifica", "Lastname": "Cognome", "Latest containers": "Ultimi contenitori", "Latest created relationships": "Ultime relazioni create", "Latitude": "Latitudine", "Layers": "Livelli", "LdapStrategy": "Strategia LDAP", + "License type": "Tipo di licenza", "Lifetime": "Durata", "Light accent": "Accento chiaro", "Light background": "Sfondo chiaro", @@ -592,6 +603,7 @@ "Locations propagation in reports": "Propagazione della posizione nei rapporti", "Log sources": "Fonti di log", "Login message": "Messaggio di login", + "Logo URI in file storage": "URI del logo nell’archivio file", "Logout remote": "Logout remoto", "Longitude": "Longitudine", "LZJD": "LZJD", @@ -607,8 +619,11 @@ "Manager contracts": "Contratti del gestore", "Manager ID": "ID manager", "Manager running": "Manager in esecuzione", + "Manager supported": "Manager supportato", "Mandatory": "Obbligatorio", "Manifest": "Manifesto", + "Manifest schema version": "Versione dello schema del manifesto", + "Manifest version": "Versione del manifesto", "Mappings": "Mappature", "Marital status": "Stato civile", "Marked for deletion at": "Contrassegnato per la cancellazione in", @@ -626,6 +641,7 @@ "Match Pir with source of relationship": "Abbinare il Pir alla fonte della relazione", "Max concurrent sessions (0 equals no maximum)": "Sessioni massime contemporanee (0 equivale a nessun massimo)", "Max Confidence": "Massima fiducia", + "Max confidence level": "Livello massimo di affidabilità", "Max policy length": "Lunghezza massima della politica", "Max shareable markings": "Marcature massime condivisibili", "Max value": "Valore massimo", @@ -807,6 +823,7 @@ "Platforms": "Piattaforme", "Playbook definition": "Definizione del playbook", "Playbook running": "Playbook in esecuzione", + "Playbook supported": "Playbook supportato", "Policy constraints": "Vincoli della policy", "Policy mappings": "Mappature della policy", "Port": "Porta", @@ -914,6 +931,7 @@ "Result": "Risultato", "Result name": "Nome del risultato", "Retention manager": "Gestore della conservazione", + "Revision": "Revisione", "Revoke score": "Revoca punteggio", "Revoked": "Revocato", "RIR": "RIR", @@ -959,6 +977,7 @@ "SHA3-256": "SHA3-256", "SHA3-512": "SHA3-512", "Shared with": "Condiviso con", + "Short description": "Breve descrizione", "Show submenu icons": "Mostra icone del sottomenu", "Sightings of observables via observed data": "Avvistamenti di osservabili tramite dati osservati", "Sightings propagation from indicator": "Propagazione degli osservabili da un indicatore", @@ -970,12 +989,15 @@ "SMTP configuration": "Configurazione SMTP", "SMTP enabled": "SMTP abilitato", "Software Vulnerability propagation to Infrastructure": "Propagazione delle vulnerabilità del software all'infrastruttura", + "Solution categories": "Categorie di soluzioni", "Sophistication": "Sofisticazione", "Source": "Fonte", + "Source code": "Codice sorgente", "Source connector": "Connettore di origine", "Source entity": "Entità di origine", "Source name": "Nome della fonte", "Source type": "Tipo di fonte", + "Source URI": "URI sorgente", "SRC": "SRC", "SRC byte count": "Conteggio byte SRC", "SRC packets": "Pacchetti SRC", @@ -1015,6 +1037,8 @@ "Subject public key exponent": "Esponente della chiave pubblica del soggetto", "Subject public key modulus": "Modulo della chiave pubblica del soggetto", "Submission date": "Data di invio", + "Subscription link": "Link di abbonamento", + "Support version": "Versione supportata", "SWID": "SWID", "Sync workflow statuses by name": "Sincronizza gli stati del flusso di lavoro in base al nome", "Synchronized": "Sincronizzato", @@ -1104,6 +1128,7 @@ "URL": "URL", "Usage propagation of parent techniques": "Propagazione d'uso delle tecniche parentali", "Usage propagation via attribution": "Propagazione dell'uso tramite attribuzione", + "Use cases": "Casi d'uso", "Use DB configuration": "Usa configurazione DB", "Use SSL": "Usa SSL", "User": "Utente", @@ -1136,6 +1161,7 @@ "Variables": "Variabili", "Vendor": "Fornitore", "verb": "verbo", + "Verified": "Verificato", "Verify SSL certificate": "Verifica del certificato SSL", "Version": "Versione", "Version History": "Storia della versione", @@ -1181,4 +1207,5 @@ "XTM One intent": "Intento di XTM One", "XTM One registration manager": "Gestore delle registrazioni XTM One", "XTM1 CGU acceptance status": "XTM1 Stato di accettazione della CGU" -} \ No newline at end of file +} + diff --git a/opencti-platform/opencti-front/lang/back/ja.json b/opencti-platform/opencti-front/lang/back/ja.json index 2439d55c6f57..b5731cf27476 100644 --- a/opencti-platform/opencti-front/lang/back/ja.json +++ b/opencti-platform/opencti-front/lang/back/ja.json @@ -108,6 +108,10 @@ "Card number": "カード番号", "Carousel order": "インデックスの日付", "Case template": "評価", + "Catalog Contract content hash": "カタログ契約の内容ハッシュ", + "Catalog Contract ID": "カタログ契約ID", + "Catalog ID": "カタログID", + "Catalog product version": "カタログ製品のバージョン", "Category": "マニフェスト", "CC": "CC", "Certifiate policies": "認証ポリシー", @@ -130,11 +134,13 @@ "Confidence level": "信頼度", "Confidentiality impact": "インシデントタイプ", "Configuration": "組織タイプ", + "Configuration schema": "構成スキーマ", "Configuration version": "モジュール", "Connector catalog": "コネクタカタログ", "Connector ID": "コネクタID", "Connector information": "コネクタ情報", "Connector manager": "コネクター・マネージャー", + "Connector manager contract": "コネクタマネージャー契約", "Connector manager current status": "コネクタマネージャの現在のステータス", "Connector manager image": "コネクタマネージャイメージ", "Connector manager requested status": "コネクタマネージャ要求ステータス", @@ -143,10 +149,12 @@ "Connector state": "コネクタの状態", "Connector trigger filters": "コネクター・トリガー・フィルター", "Connector type": "コネクタタイプ", + "Connector upgrade strategy": "コネクタのアップグレード戦略", "Connector user id": "コネクター・ユーザーID", "Connectors": "コネクタ", "Consent confirm text": "同意確認テキスト", "Consent message": "同意メッセージ", + "Contact": "連絡先", "Contact information": "連絡先情報", "Contains": "含む", "Contains observable": "観測値を含む", @@ -160,6 +168,7 @@ "Context": "コンテキスト", "Context data": "コンテキストデータ", "Contextual only": "コンテキストのみ", + "Contract version": "契約バージョン", "Copy confidence level to OpenCTI scores for indicators": "指標の信頼度をOpenCTIのスコアにコピーする", "Copy OpenCTI scores to confidence level for indicators": "OpenCTIのスコアを指標の信頼度にコピーする", "Count": "偽陽性", @@ -562,12 +571,14 @@ "Last score evolution": "最終スコアの推移", "Last seen": "目的", "Last update": "最終更新日", + "Last verified date": "最終検証日", "Lastname": "姓", "Latest containers": "最新コンテナ", "Latest created relationships": "最新作成リレーションシップ", "Latitude": "緯度", "Layers": "6 - 真実は判断できない", "LdapStrategy": "LDAP戦略", + "License type": "ライセンスタイプ", "Lifetime": "生涯", "Light accent": "ライトアクセント", "Light background": "明るい背景", @@ -592,6 +603,7 @@ "Locations propagation in reports": "レポートにおける位置情報の伝播", "Log sources": "インフラの種類", "Login message": "ログインメッセージ", + "Logo URI in file storage": "ファイルストレージ内のロゴ URI", "Logout remote": "ログアウト・リモート", "Longitude": "経度", "LZJD": "LZJD", @@ -607,8 +619,11 @@ "Manager contracts": "マネージャー契約", "Manager ID": "設定", "Manager running": "マネージャー実行中", + "Manager supported": "対応マネージャー", "Mandatory": "必須", "Manifest": "タグ", + "Manifest schema version": "マニフェストスキーマバージョン", + "Manifest version": "マニフェストバージョン", "Mappings": "マッピング", "Marital status": "目の色", "Marked for deletion at": "削除対象としてマークされました", @@ -626,6 +641,7 @@ "Match Pir with source of relationship": "海賊と関係元を一致させる", "Max concurrent sessions (0 equals no maximum)": "最大同時セッション数（0は最大なし）", "Max Confidence": "最大信頼度", + "Max confidence level": "最大信頼度", "Max policy length": "最大ポリシー長", "Max shareable markings": "共有可能な最大マーキング", "Max value": "最大値", @@ -807,6 +823,7 @@ "Platforms": "必要なパーミッション", "Playbook definition": "プレイブックの定義", "Playbook running": "プレイブックの実行", + "Playbook supported": "対応プレイブック", "Policy constraints": "ポリシー制約", "Policy mappings": "ポリシーのマッピング", "Port": "ポート", @@ -914,6 +931,7 @@ "Result": "VMホスト", "Result name": "分析結果", "Retention manager": "リテンション・マネージャー", + "Revision": "リビジョン", "Revoke score": "失効点数", "Revoked": "ラング", "RIR": "RIR", @@ -959,6 +977,7 @@ "SHA3-256": "SHA3-256", "SHA3-512": "SHA3-512", "Shared with": "と共有", + "Short description": "概要", "Show submenu icons": "サブメニューアイコンを表示する", "Sightings of observables via observed data": "観測データを介した観測可能性の目撃", "Sightings propagation from indicator": "指標からの目撃伝播", @@ -970,12 +989,15 @@ "SMTP configuration": "SMTP設定", "SMTP enabled": "SMTPを有効にする", "Software Vulnerability propagation to Infrastructure": "ソフトウェアの脆弱性がインフラストラクチャに波及する", + "Solution categories": "ソリューションカテゴリ", "Sophistication": "個人的な動機", "Source": "有効", + "Source code": "ソースコード", "Source connector": "ソースコネクター", "Source entity": "ソースエンティティ", "Source name": "ハッシュ", "Source type": "ソースタイプ", + "Source URI": "ソースURI", "SRC": "SRC", "SRC byte count": "SRCバイト数", "SRC packets": "SRCパケット数", @@ -1015,6 +1037,8 @@ "Subject public key exponent": "主題の公開鍵の指数", "Subject public key modulus": "対象公開鍵モジュラス", "Submission date": "分析開始日", + "Subscription link": "サブスクリプションリンク", + "Support version": "サポート対象バージョン", "SWID": "SWID", "Sync workflow statuses by name": "ワークフローのステータスを名前で同期する", "Synchronized": "同期", @@ -1104,6 +1128,7 @@ "URL": "URL", "Usage propagation of parent techniques": "親テクニックの利用伝播", "Usage propagation via attribution": "アトリビューションによる利用伝播", + "Use cases": "ユースケース", "Use DB configuration": "DB設定を使用する", "Use SSL": "SSLを使用する", "User": "内蔵", @@ -1136,6 +1161,7 @@ "Variables": "変数", "Vendor": "ベンダー", "verb": "動詞", + "Verified": "検証済み", "Verify SSL certificate": "SSL証明書の検証", "Version": "バージョン", "Version History": "バージョン履歴", @@ -1181,4 +1207,5 @@ "XTM One intent": "XTMワンの意図", "XTM One registration manager": "XTM One登録マネージャー", "XTM1 CGU acceptance status": "XTM1 CGU受入状況" -} \ No newline at end of file +} + diff --git a/opencti-platform/opencti-front/lang/back/ko.json b/opencti-platform/opencti-front/lang/back/ko.json index 816ca35c4723..bbc321d3c5db 100644 --- a/opencti-platform/opencti-front/lang/back/ko.json +++ b/opencti-platform/opencti-front/lang/back/ko.json @@ -108,6 +108,10 @@ "Card number": "카드 번호", "Carousel order": "캐러셀 순서", "Case template": "사례 템플릿", + "Catalog Contract content hash": "카탈로그 계약 내용 해시", + "Catalog Contract ID": "카탈로그 계약 ID", + "Catalog ID": "카탈로그 ID", + "Catalog product version": "카탈로그 제품 버전", "Category": "카테고리", "CC": "CC", "Certifiate policies": "인증서 정책", @@ -130,11 +134,13 @@ "Confidence level": "신뢰 수준", "Confidentiality impact": "기밀성 영향", "Configuration": "구성", + "Configuration schema": "구성 스키마", "Configuration version": "구성 버전", "Connector catalog": "커넥터 카탈로그", "Connector ID": "커넥터 ID", "Connector information": "커넥터 정보", "Connector manager": "커넥터 관리자", + "Connector manager contract": "커넥터 관리자 계약", "Connector manager current status": "커넥터 관리자 현재 상태", "Connector manager image": "커넥터 관리자 이미지", "Connector manager requested status": "커넥터 관리자 요청 상태", @@ -143,10 +149,12 @@ "Connector state": "커넥터 상태", "Connector trigger filters": "커넥터 트리거 필터", "Connector type": "커넥터 유형", + "Connector upgrade strategy": "커넥터 업그레이드 전략", "Connector user id": "커넥터 사용자 ID", "Connectors": "커넥터", "Consent confirm text": "동의 확인 텍스트", "Consent message": "동의 메시지", + "Contact": "연락처", "Contact information": "연락처 정보", "Contains": "포함", "Contains observable": "관찰 가능 항목 포함", @@ -160,6 +168,7 @@ "Context": "컨텍스트", "Context data": "컨텍스트 데이터", "Contextual only": "컨텍스트 전용", + "Contract version": "계약 버전", "Copy confidence level to OpenCTI scores for indicators": "지표에 대한 신뢰 수준을 OpenCTI 점수로 복사합니다", "Copy OpenCTI scores to confidence level for indicators": "지표의 신뢰 수준에 OpenCTI 점수를 복사합니다", "Count": "개수", @@ -562,12 +571,14 @@ "Last score evolution": "마지막 점수 진화", "Last seen": "마지막 목격", "Last update": "마지막 업데이트", + "Last verified date": "최종 검증 날짜", "Lastname": "성", "Latest containers": "최신 컨테이너", "Latest created relationships": "최근에 생성된 관계", "Latitude": "위도", "Layers": "레이어", "LdapStrategy": "LDAP 전략", + "License type": "라이선스 유형", "Lifetime": "수명", "Light accent": "밝은 강조", "Light background": "밝은 배경", @@ -592,6 +603,7 @@ "Locations propagation in reports": "보고서 내 위치 전파", "Log sources": "로그 소스", "Login message": "로그인 메시지", + "Logo URI in file storage": "파일 저장소의 로고 URI", "Logout remote": "원격 로그아웃", "Longitude": "경도", "LZJD": "LZJD", @@ -607,8 +619,11 @@ "Manager contracts": "관리자 컨트랙트", "Manager ID": "관리자 ID", "Manager running": "관리자 실행 중", + "Manager supported": "지원되는 관리자", "Mandatory": "필수", "Manifest": "매니페스트", + "Manifest schema version": "매니페스트 스키마 버전", + "Manifest version": "매니페스트 버전", "Mappings": "매핑", "Marital status": "결혼 상태", "Marked for deletion at": "다음 위치에서 삭제 대상으로 표시됨", @@ -626,6 +641,7 @@ "Match Pir with source of relationship": "Pir와 관계 소스 일치", "Max concurrent sessions (0 equals no maximum)": "최대 동시 세션(0은 최대값 없음)", "Max Confidence": "최대 신뢰도", + "Max confidence level": "최대 신뢰 수준", "Max policy length": "최대 정책 길이", "Max shareable markings": "최대 공유 가능 마킹", "Max value": "최대값", @@ -807,6 +823,7 @@ "Platforms": "플랫폼", "Playbook definition": "플레이북 정의", "Playbook running": "플레이북 실행 중", + "Playbook supported": "지원되는 플레이북", "Policy constraints": "정책 제약 조건", "Policy mappings": "정책 매핑", "Port": "포트", @@ -914,6 +931,7 @@ "Result": "결과", "Result name": "결과 이름", "Retention manager": "리텐션 관리자", + "Revision": "개정판", "Revoke score": "점수 취소", "Revoked": "취소됨", "RIR": "RIR", @@ -959,6 +977,7 @@ "SHA3-256": "SHA3-256", "SHA3-512": "SHA3-512", "Shared with": "공유됨", + "Short description": "간략한 설명", "Show submenu icons": "서브메뉴 아이콘 표시", "Sightings of observables via observed data": "관측 데이터를 통한 관측 가능 항목 발견", "Sightings propagation from indicator": "인디케이터를 통한 관측값 전파", @@ -970,12 +989,15 @@ "SMTP configuration": "SMTP 구성", "SMTP enabled": "SMTP 사용", "Software Vulnerability propagation to Infrastructure": "인프라로 전파되는 소프트웨어 취약점", + "Solution categories": "솔루션 카테고리", "Sophistication": "정교함", "Source": "출처", + "Source code": "소스 코드", "Source connector": "출처 커넥터", "Source entity": "출처 엔터티", "Source name": "출처 이름", "Source type": "출처 유형", + "Source URI": "소스 URI", "SRC": "SRC", "SRC byte count": "SRC 바이트 수", "SRC packets": "SRC 패킷", @@ -1015,6 +1037,8 @@ "Subject public key exponent": "주제 공개 키 지수", "Subject public key modulus": "주제 공개 키 모듈러스", "Submission date": "제출 날짜", + "Subscription link": "구독 링크", + "Support version": "지원 버전", "SWID": "SWID", "Sync workflow statuses by name": "이름별로 워크플로 상태 동기화", "Synchronized": "동기화됨", @@ -1104,6 +1128,7 @@ "URL": "URL", "Usage propagation of parent techniques": "상위 기술의 사용 전파", "Usage propagation via attribution": "어트리뷰션을 통한 사용량 전파", + "Use cases": "사용 사례", "Use DB configuration": "DB 구성 사용", "Use SSL": "SSL 사용", "User": "사용자", @@ -1136,6 +1161,7 @@ "Variables": "변수", "Vendor": "공급업체", "verb": "동사", + "Verified": "검증됨", "Verify SSL certificate": "SSL 인증서 확인", "Version": "버전", "Version History": "버전 기록", @@ -1181,4 +1207,5 @@ "XTM One intent": "XTM One 의도", "XTM One registration manager": "XTM One 등록 관리자", "XTM1 CGU acceptance status": "XTM1 CGU 승인 상태" -} \ No newline at end of file +} + diff --git a/opencti-platform/opencti-front/lang/back/ru.json b/opencti-platform/opencti-front/lang/back/ru.json index dde2ed666f5a..721e4a465d7f 100644 --- a/opencti-platform/opencti-front/lang/back/ru.json +++ b/opencti-platform/opencti-front/lang/back/ru.json @@ -108,6 +108,10 @@ "Card number": "Номер карты", "Carousel order": "Порядок в карусели", "Case template": "Шаблон кейса", + "Catalog Contract content hash": "Хеш содержимого контракта каталога", + "Catalog Contract ID": "Идентификатор контракта каталога", + "Catalog ID": "Идентификатор каталога", + "Catalog product version": "Версия продукта в каталоге", "Category": "Категория", "CC": "CC", "Certifiate policies": "Политики сертификации", @@ -130,11 +134,13 @@ "Confidence level": "Уровень доверия", "Confidentiality impact": "Влияние на конфиденциальность", "Configuration": "Конфигурация", + "Configuration schema": "Схема конфигурации", "Configuration version": "Версия конфигурации", "Connector catalog": "Каталог коннекторов", "Connector ID": "Идентификатор коннектора", "Connector information": "Информация о коннекторе", "Connector manager": "Менеджер коннектора", + "Connector manager contract": "Контракт диспетчера коннекторов", "Connector manager current status": "Текущее состояние менеджера коннектора", "Connector manager image": "Изображение менеджера коннекторов", "Connector manager requested status": "Запрошенное состояние менеджера коннектора", @@ -143,10 +149,12 @@ "Connector state": "Состояние коннектора", "Connector trigger filters": "Триггерные фильтры коннектора", "Connector type": "Тип коннектора", + "Connector upgrade strategy": "Стратегия обновления коннекторов", "Connector user id": "Идентификатор пользователя коннектора", "Connectors": "Коннекторы", "Consent confirm text": "Согласие подтвердить текст", "Consent message": "Сообщение о согласии", + "Contact": "Контактные данные", "Contact information": "Контактная информация", "Contains": "Содержит", "Contains observable": "Содержит наблюдаемые", @@ -160,6 +168,7 @@ "Context": "Контекст", "Context data": "Контекстные данные", "Contextual only": "Только контекст", + "Contract version": "Версия контракта", "Copy confidence level to OpenCTI scores for indicators": "Копирование уровня доверия к показателям OpenCTI", "Copy OpenCTI scores to confidence level for indicators": "Скопируйте оценки OpenCTI в уровень доверия для индикаторов", "Count": "Количество", @@ -562,12 +571,14 @@ "Last score evolution": "Эволюция последней оценки", "Last seen": "Последний раз видели", "Last update": "Последнее обновление", + "Last verified date": "Дата последней проверки", "Lastname": "Фамилия", "Latest containers": "Последние контейнеры", "Latest created relationships": "Последние созданные отношения", "Latitude": "Широта", "Layers": "Слои", "LdapStrategy": "LdapStrategy", + "License type": "Тип лицензии", "Lifetime": "Пожизненный", "Light accent": "Светлый акцент", "Light background": "Светлый фон", @@ -592,6 +603,7 @@ "Locations propagation in reports": "Распространение местоположений в отчетах", "Log sources": "Источники журналов", "Login message": "Сообщение о входе в систему", + "Logo URI in file storage": "URI логотипа в хранилище файлов", "Logout remote": "Удаленный выход из системы", "Longitude": "Долгота", "LZJD": "LZJD", @@ -607,8 +619,11 @@ "Manager contracts": "Договоры с менеджерами", "Manager ID": "Идентификатор менеджера", "Manager running": "Менеджер работает", + "Manager supported": "Поддерживаемый менеджер", "Mandatory": "Обязательное поле", "Manifest": "Манифест", + "Manifest schema version": "Версия схемы манифеста", + "Manifest version": "Версия манифеста", "Mappings": "Отображения", "Marital status": "Семейное положение", "Marked for deletion at": "Отмечено для удаления в", @@ -626,6 +641,7 @@ "Match Pir with source of relationship": "Сопоставьте Пир с источником отношений", "Max concurrent sessions (0 equals no maximum)": "Максимальное количество одновременных сеансов (0 означает отсутствие максимума)", "Max Confidence": "Максимальная уверенность", + "Max confidence level": "Максимальный уровень достоверности", "Max policy length": "Максимальная длина полиса", "Max shareable markings": "Максимально разделяемые метки", "Max value": "Максимальное значение", @@ -807,6 +823,7 @@ "Platforms": "Платформы", "Playbook definition": "Определение игровой книги", "Playbook running": "Работа с игровыми книгами", + "Playbook supported": "Поддерживаемые руководства", "Policy constraints": "Политические ограничения", "Policy mappings": "Сопоставление политик", "Port": "Порт", @@ -914,6 +931,7 @@ "Result": "Результат", "Result name": "Название результата", "Retention manager": "Менеджер по удержанию персонала", + "Revision": "Ревизия", "Revoke score": "Отменить оценку", "Revoked": "Отозвано", "RIR": "RIR", @@ -959,6 +977,7 @@ "SHA3-256": "SHA3-256", "SHA3-512": "SHA3-512", "Shared with": "Разделена с", + "Short description": "Краткое описание", "Show submenu icons": "Показать значки подменю", "Sightings of observables via observed data": "Наблюдения наблюдаемых через наблюдаемые данные", "Sightings propagation from indicator": "Распространение наблюдений через индикатор", @@ -970,12 +989,15 @@ "SMTP configuration": "Конфигурация SMTP", "SMTP enabled": "SMTP включен", "Software Vulnerability propagation to Infrastructure": "Распространение уязвимостей программного обеспечения на инфраструктуру", + "Solution categories": "Категории решений", "Sophistication": "Изысканность", "Source": "Источник", + "Source code": "Исходный код", "Source connector": "Исходный коннектор", "Source entity": "Источник", "Source name": "Название источника", "Source type": "Тип источника", + "Source URI": "URI источника", "SRC": "SRC", "SRC byte count": "Счетчик байтов SRC", "SRC packets": "Пакеты SRC", @@ -1015,6 +1037,8 @@ "Subject public key exponent": "Экспонента открытого ключа субъекта", "Subject public key modulus": "Модуль открытого ключа субъекта", "Submission date": "Дата представления", + "Subscription link": "Ссылка на подписку", + "Support version": "Поддерживаемая версия", "SWID": "SWID", "Sync workflow statuses by name": "Синхронизация статусов рабочих процессов по названию", "Synchronized": "Синхронизированный", @@ -1104,6 +1128,7 @@ "URL": "URL", "Usage propagation of parent techniques": "Распространение использования родительских техник", "Usage propagation via attribution": "Распространение использования через атрибуцию", + "Use cases": "Варианты использования", "Use DB configuration": "Использовать настройки БД", "Use SSL": "Использовать SSL", "User": "Пользователь", @@ -1136,6 +1161,7 @@ "Variables": "Переменные", "Vendor": "Поставщик", "verb": "глагол", + "Verified": "Проверено", "Verify SSL certificate": "Проверка сертификата SSL", "Version": "Версия", "Version History": "История версий", @@ -1181,4 +1207,5 @@ "XTM One intent": "Намерение XTM One", "XTM One registration manager": "Менеджер регистрации XTM One", "XTM1 CGU acceptance status": "Статус принятия XTM1 CGU" -} \ No newline at end of file +} + diff --git a/opencti-platform/opencti-front/lang/back/zh.json b/opencti-platform/opencti-front/lang/back/zh.json index a1011d4c0fa8..56780f82d937 100644 --- a/opencti-platform/opencti-front/lang/back/zh.json +++ b/opencti-platform/opencti-front/lang/back/zh.json @@ -108,6 +108,10 @@ "Card number": "卡号", "Carousel order": "传送带顺序", "Case template": "案例模板", + "Catalog Contract content hash": "目录合同内容哈希", + "Catalog Contract ID": "目录合同 ID", + "Catalog ID": "目录 ID", + "Catalog product version": "目录产品版本", "Category": "类别", "CC": "抄送", "Certifiate policies": "认证策略", @@ -130,11 +134,13 @@ "Confidence level": "置信度", "Confidentiality impact": "保密性影响", "Configuration": "配置", + "Configuration schema": "配置模式", "Configuration version": "配置版本", "Connector catalog": "连接器目录", "Connector ID": "连接器 ID", "Connector information": "连接器信息", "Connector manager": "连接器管理器", + "Connector manager contract": "连接器管理器合同", "Connector manager current status": "连接器管理器当前状态", "Connector manager image": "连接器管理器图像", "Connector manager requested status": "连接器管理器请求状态", @@ -143,10 +149,12 @@ "Connector state": "连接器状态", "Connector trigger filters": "连接器触发滤波器", "Connector type": "连接器类型", + "Connector upgrade strategy": "连接器升级策略", "Connector user id": "连接器用户 ID", "Connectors": "连接器", "Consent confirm text": "同意确认文本", "Consent message": "同意信息", + "Contact": "联系方式", "Contact information": "联系信息", "Contains": "包含", "Contains observable": "包含观测值", @@ -160,6 +168,7 @@ "Context": "上下文", "Context data": "上下文数据", "Contextual only": "仅根据上下文", + "Contract version": "合同版本", "Copy confidence level to OpenCTI scores for indicators": "将置信度复制到指标的 OpenCTI 分数", "Copy OpenCTI scores to confidence level for indicators": "将 OpenCTI 分数复制到指标的置信度", "Count": "计数", @@ -562,12 +571,14 @@ "Last score evolution": "最后分数演变", "Last seen": "最近出现", "Last update": "最后更新", + "Last verified date": "最后验证日期", "Lastname": "姓氏", "Latest containers": "最新容器", "Latest created relationships": "最近创建的关系", "Latitude": "纬度", "Layers": "层级", "LdapStrategy": "LDAP Strategy", + "License type": "许可证类型", "Lifetime": "生命周期", "Light accent": "淡色调", "Light background": "淡色背景", @@ -592,6 +603,7 @@ "Locations propagation in reports": "报告中的位置传播", "Log sources": "日志来源", "Login message": "登录信息", + "Logo URI in file storage": "文件存储中的徽标 URI", "Logout remote": "远程注销", "Longitude": "经度", "LZJD": "LZJD", @@ -607,8 +619,11 @@ "Manager contracts": "管理器合同", "Manager ID": "管理者 ID", "Manager running": "管理器运行", + "Manager supported": "支持的管理器", "Mandatory": "必填", "Manifest": "内容", + "Manifest schema version": "清单架构版本", + "Manifest version": "清单版本", "Mappings": "映射", "Marital status": "婚姻状况", "Marked for deletion at": "已标记为删除，位置在", @@ -626,6 +641,7 @@ "Match Pir with source of relationship": "将 Pir 与关系源匹配", "Max concurrent sessions (0 equals no maximum)": "最大并发会话数（0 表示无最大值）", "Max Confidence": "最大置信度", + "Max confidence level": "最大置信度", "Max policy length": "最多长度要求", "Max shareable markings": "最大可共享标记", "Max value": "最大值", @@ -807,6 +823,7 @@ "Platforms": "平台", "Playbook definition": "游戏本定义", "Playbook running": "运行游戏本", + "Playbook supported": "支持的剧本", "Policy constraints": "策略约束", "Policy mappings": "策略映射", "Port": "端口", @@ -914,6 +931,7 @@ "Result": "结果", "Result name": "结果名称", "Retention manager": "留用经理", + "Revision": "修订版", "Revoke score": "撤销得分", "Revoked": "撤销", "RIR": "RIR", @@ -959,6 +977,7 @@ "SHA3-256": "SHA3-256", "SHA3-512": "SHA3-512", "Shared with": "共享对象", + "Short description": "简要描述", "Show submenu icons": "显示子菜单图标", "Sightings of observables via observed data": "通过观察数据看到可观察数据", "Sightings propagation from indicator": "从指标传播观察结果", @@ -970,12 +989,15 @@ "SMTP configuration": "SMTP配置", "SMTP enabled": "已启用 SMTP", "Software Vulnerability propagation to Infrastructure": "软件漏洞向基础设施的传播", + "Solution categories": "解决方案类别", "Sophistication": "复杂性", "Source": "来源", + "Source code": "源代码", "Source connector": "源连接器", "Source entity": "来源实体", "Source name": "源名称", "Source type": "源类型", + "Source URI": "源代码 URI", "SRC": "SRC", "SRC byte count": "SRC 字节数", "SRC packets": "SRC 数据包", @@ -1015,6 +1037,8 @@ "Subject public key exponent": "主体公钥指数", "Subject public key modulus": "主体公钥模数", "Submission date": "提交日期", + "Subscription link": "订阅链接", + "Support version": "支持版本", "SWID": "SWID", "Sync workflow statuses by name": "按名称同步工作流状态", "Synchronized": "同步", @@ -1104,6 +1128,7 @@ "URL": "URL", "Usage propagation of parent techniques": "父技术的使用传播", "Usage propagation via attribution": "通过归属进行使用传播", + "Use cases": "用例", "Use DB configuration": "使用数据库配置", "Use SSL": "使用 SSL", "User": "用户", @@ -1136,6 +1161,7 @@ "Variables": "变量", "Vendor": "供应商", "verb": "动词", + "Verified": "已验证", "Verify SSL certificate": "验证 SSL 证书", "Version": "版本", "Version History": "版本历史", @@ -1181,4 +1207,5 @@ "XTM One intent": "XTM 一个意图", "XTM One registration manager": "XTM One 注册管理器", "XTM1 CGU acceptance status": "XTM1 CGU 验收状态" -} \ No newline at end of file +} + From e0c169e606f4666d663629cd2aad9edf063ecf19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Wed, 19 Aug 2026 18:59:35 +0200 Subject: [PATCH 19/45] fix(catalog): use structured errors for logo storage failures (#0) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../modules/catalog/catalog-logo-storage.ts | 45 ++++++++++++++----- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-logo-storage.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-logo-storage.ts index df047d04aa7f..c9022bfe962b 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-logo-storage.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-logo-storage.ts @@ -3,6 +3,7 @@ import { rawListObjects, rawUpload } from '../../database/raw-file-storage'; import { parseDataUrl } from '../../utils/data-url'; import type { CatalogContract } from './catalog-types'; import { CATALOG_LOGO_VIEW_PATH } from './catalog-http'; +import { FilesystemError, UnknownError, UnsupportedError } from '../../config/errors'; export const CATALOG_CONTRACT_LOGOS_DIR = 'catalog-logos'; @@ -77,7 +78,10 @@ export const storeCatalogContractLogo = async ( return operationResult; } if (!operationResult.operation) { - const error = new Error('Missing logo upload operation'); + const error = UnknownError('Missing logo upload operation', { + contractTitle: contractDto.title, + contractSlug: contractDto.slug, + }); return { result: 'failed' as const, logoUri: null, @@ -113,7 +117,10 @@ export const computeCatalogContractLogoUploadOperation = ( }; } if (!logo.startsWith('data:')) { - const error = new Error('Unsupported logo data format: not a data URL'); + const error = UnsupportedError('Unsupported logo data format: not a data URL', { + contractTitle: contractDto.title, + contractSlug: contractDto.slug, + }); return { result: 'failed' as const, logoUri: null, @@ -122,10 +129,10 @@ export const computeCatalogContractLogoUploadOperation = ( } const { mimeType, base64Encoded, data } = parseDataUrl(logo); if (!isImageMimeType(mimeType)) { - const error = new Error('Unsupported logo mime type', { - cause: { - mimeType, - }, + const error = UnsupportedError('Unsupported logo mime type', { + contractTitle: contractDto.title, + contractSlug: contractDto.slug, + mimeType, }); return { result: 'failed' as const, @@ -136,7 +143,10 @@ export const computeCatalogContractLogoUploadOperation = ( let decodedData: Buffer; if (base64Encoded) { if (!isLikelyBase64(data)) { - const error = new Error('Unsupported logo data format: invalid base64 payload'); + const error = UnsupportedError('Unsupported logo data format: invalid base64 payload', { + contractTitle: contractDto.title, + contractSlug: contractDto.slug, + }); return { result: 'failed' as const, logoUri: null, @@ -148,7 +158,9 @@ export const computeCatalogContractLogoUploadOperation = ( try { decodedData = Buffer.from(decodeURIComponent(data), 'utf8'); } catch (err: unknown) { - const error = new Error('Unsupported logo data format: invalid URL-encoded payload', { + const error = UnsupportedError('Unsupported logo data format: invalid URL-encoded payload', { + contractTitle: contractDto.title, + contractSlug: contractDto.slug, cause: err, }); return { @@ -159,7 +171,10 @@ export const computeCatalogContractLogoUploadOperation = ( } } if (decodedData.byteLength === 0) { - const error = new Error('Unexpected zero-length logo data'); + const error = UnsupportedError('Unexpected zero-length logo data', { + contractTitle: contractDto.title, + contractSlug: contractDto.slug, + }); return { result: 'failed' as const, logoUri: null, @@ -167,7 +182,12 @@ export const computeCatalogContractLogoUploadOperation = ( }; } if (decodedData.byteLength > CATALOG_CONTRACT_LOGO_MAX_SIZE_BYTES) { - const error = new Error(`Unsupported logo data format: image exceeds max size (${CATALOG_CONTRACT_LOGO_MAX_SIZE_BYTES} bytes)`); + const error = UnsupportedError('Unsupported logo data format: image exceeds max size', { + contractTitle: contractDto.title, + contractSlug: contractDto.slug, + maxSizeBytes: CATALOG_CONTRACT_LOGO_MAX_SIZE_BYTES, + actualSizeBytes: decodedData.byteLength, + }); return { result: 'failed' as const, logoUri: null, @@ -206,7 +226,10 @@ export const uploadCatalogContractLogoOperation = async (operation: CatalogContr try { await rawUpload(operation.s3Key, operation.body); } catch (err: unknown) { - const error = new Error('Failed to upload logo content to file storage', { + const error = FilesystemError('Failed to upload logo content to file storage', { + s3Key: operation.s3Key, + filename: operation.filename, + logoUri: operation.logoUri, cause: err, }); return { From 99179fadbdf9a654f25cfeef90a6563e67ab3ecd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Wed, 19 Aug 2026 19:00:39 +0200 Subject: [PATCH 20/45] fix(backend): remove inspect flag from default start script (#0) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- opencti-platform/opencti-graphql/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opencti-platform/opencti-graphql/package.json b/opencti-platform/opencti-graphql/package.json index 6fe115887043..dc389b1d8698 100644 --- a/opencti-platform/opencti-graphql/package.json +++ b/opencti-platform/opencti-graphql/package.json @@ -17,7 +17,7 @@ "frontend:stack:analysis": "node ./script/script-stack-analysis front", "lint": "DEBUG=eslint:cli-engine TIMING=1 eslint --quiet --cache --config eslint.config.js src", "build": "run install:python && run build:prod", - "start": "run build:dev && node --enable-source-maps --inspect build/back.mjs", + "start": "run build:dev && node --enable-source-maps build/back.mjs", "start:e2e": "run build:dev && APP__BASE_PATH=/e2e APP__ENABLED_DEV_FEATURES='[\"*\"]' EXPIRATION_SCHEDULER__ENABLED=false REDIS__NAMESPACE=e2e-start RABBITMQ__QUEUE_PREFIX=e2e MINIO__BUCKET_NAME=e2e-start-bucket ELASTICSEARCH__INDEX_PREFIX=e2e-start node build/back.mjs", "start:light": "NODE_ENV=development node --enable-source-maps build/back.mjs", "start:cluster": "NODE_ENV=cluster node --enable-source-maps build/back.mjs", From 29a5dc6ebc66be81861f3ab947c37c5087bae867 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Wed, 19 Aug 2026 19:04:43 +0200 Subject: [PATCH 21/45] revert(core): rollback score schema move and manager delay option (#0) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../opencti-front/src/schema/relay.schema.graphql | 4 ++-- opencti-platform/opencti-graphql/src/manager/managerModule.ts | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/opencti-platform/opencti-front/src/schema/relay.schema.graphql b/opencti-platform/opencti-front/src/schema/relay.schema.graphql index 3edda2c6b229..291536faf5ea 100644 --- a/opencti-platform/opencti-front/src/schema/relay.schema.graphql +++ b/opencti-platform/opencti-front/src/schema/relay.schema.graphql @@ -5585,9 +5585,9 @@ enum ThreatActorsOrdering { updated_at x_opencti_workflow_id confidence - x_opencti_score sophistication resource_level + x_opencti_score _score threat_actor_types objectMarking @@ -5696,7 +5696,6 @@ type ThreatActorGroup implements BasicObject & StixObject & StixCoreObject & Sti metrics: [Metric] representative: Representative! x_opencti_stix_ids: [StixId] - x_opencti_score: Int is_inferred: Boolean! spec_version: String! created_at: DateTime! @@ -5745,6 +5744,7 @@ type ThreatActorGroup implements BasicObject & StixObject & StixCoreObject & Sti goals: [String] sophistication: String resource_level: String + x_opencti_score: Int primary_motivation: String secondary_motivations: [String] personal_motivations: [String] diff --git a/opencti-platform/opencti-graphql/src/manager/managerModule.ts b/opencti-platform/opencti-graphql/src/manager/managerModule.ts index d990e064005f..ee8183cbf1e1 100644 --- a/opencti-platform/opencti-graphql/src/manager/managerModule.ts +++ b/opencti-platform/opencti-graphql/src/manager/managerModule.ts @@ -20,7 +20,6 @@ export interface HandlerInput { export interface ManagerCronScheduler { handler: (input?: any) => Promise; shutdown?: () => void; - delayToStart?: number; // Defaults to `interval` interval: number; lockKey: string; infiniteInterval?: number; From 4322643cd82efdcf4228276dbb3f24e9f90c1dca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Wed, 19 Aug 2026 19:10:41 +0200 Subject: [PATCH 22/45] refactor(catalog): move sync timeout config to manager Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../src/modules/catalog/catalog-manager.ts | 13 +++++- .../catalog/sync/catalog-sync-domain.ts | 32 +++++++-------- .../sync/catalog-sync-source-gateway.ts | 40 +++++++++++++------ .../catalog/catalog-sync-domain-test.ts | 6 +-- .../catalog-sync-source-gateway-test.ts | 6 +++ 5 files changed, 65 insertions(+), 32 deletions(-) diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-manager.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-manager.ts index 5b6b4c3a0a18..0b2b0be16dd8 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-manager.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-manager.ts @@ -10,13 +10,24 @@ const CATALOG_MANAGER_CONTEXT = 'catalog_manager'; const CATALOG_MANAGER_ENABLED = booleanConf('catalog_manager:enabled', true); const CATALOG_MANAGER_LOCK_KEY = conf.get('catalog_manager:lock_key') ?? 'catalog_manager_lock'; const CATALOG_MANAGER_INTERVAL = conf.get('catalog_manager:interval') ?? 60_000; +const DEFAULT_CATALOG_SYNC_REMOTE_TIMEOUT_MS = 30_000; + +const resolveCatalogSyncRemoteTimeoutMs = () => { + const configuredTimeout = Number(conf.get('catalog_manager:catalog_sync_remote_timeout') ?? conf.get('app:request_timeout')); + if (!Number.isFinite(configuredTimeout) || configuredTimeout <= 0) { + return DEFAULT_CATALOG_SYNC_REMOTE_TIMEOUT_MS; + } + return configuredTimeout; +}; export const isCatalogManagerEnabled = () => CATALOG_MANAGER_ENABLED; const catalogManagerHandler = async () => { const context = executionContext('catalog_manager'); // Sync catalogs to ES - const syncedCatalogsWithChanges = await synchronizeCatalogs(context, SYSTEM_USER); + const syncedCatalogsWithChanges = await synchronizeCatalogs(context, SYSTEM_USER, { + remoteCatalogTimeoutMs: resolveCatalogSyncRemoteTimeoutMs(), + }); // Apply upgrade strategy. // Should be moved another manager maybe (connectorManager). // Consider using an event to invert the dependency. diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts b/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts index aa853f719fe1..3d53446d0a1c 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts @@ -32,7 +32,7 @@ import { upsertCatalog, } from '../catalog-repository'; import type { CatalogContractSyncSource, CatalogSyncSource, CatalogSyncSourceConfig } from './catalog-sync-types'; -import { fetchSourceCatalog, fetchSourceCatalogRevisionHint } from './catalog-sync-source-gateway'; +import { type CatalogSyncSourceGatewayOptions, fetchSourceCatalog, fetchSourceCatalogRevisionHint } from './catalog-sync-source-gateway'; const DECOUPLING_VERSIONS_FEATURE_FLAG = 'DECOUPLING_VERSIONS'; @@ -227,7 +227,12 @@ const computeCatalogSyncOps = (params: { }; }; -const synchronizeCatalog = async (context: AuthContext, user: AuthUser, sourceConfig: CatalogSyncSourceConfig) => { +const synchronizeCatalog = async ( + context: AuthContext, + user: AuthUser, + sourceConfig: CatalogSyncSourceConfig, + options?: CatalogSyncSourceGatewayOptions, +) => { try { logApp.debug('[OPENCTI-MODULE] Synchronizing catalog', { sourceKind: sourceConfig.kind, @@ -237,7 +242,7 @@ const synchronizeCatalog = async (context: AuthContext, user: AuthUser, sourceCo ? await findCatalogBySourceUri(context, user, sourceConfig.uri) : undefined; const remoteRevisionHint = sourceConfig.kind === 'remote' - ? await fetchSourceCatalogRevisionHint(sourceConfig) + ? await fetchSourceCatalogRevisionHint(sourceConfig, options) : undefined; if (sourceConfig.kind === 'remote' && currentCatalogBySourceUri?.revision) { if (remoteRevisionHint && remoteRevisionHint === currentCatalogBySourceUri.revision) { @@ -254,7 +259,7 @@ const synchronizeCatalog = async (context: AuthContext, user: AuthUser, sourceCo } } // Fetch source catalog - const sourceCatalog = await fetchSourceCatalog(sourceConfig); + const sourceCatalog = await fetchSourceCatalog(sourceConfig, options); // Find existing persisted data in the database corresponding to the catalog id const currentCatalog = await findCatalogByCatalogId(context, user, sourceCatalog.id); const currentRevision = currentCatalog?.revision; @@ -315,12 +320,7 @@ const synchronizeCatalog = async (context: AuthContext, user: AuthUser, sourceCo }); // Persist refreshed catalog, contracts & logos // Something's fishy if this occurs too frequently. -<<<<<<< HEAD logApp.warn('[OPENCTI-MODULE] Persisting catalog & contracts', { -======= - logCatalog.debug('[OPENCTI-MODULE] Persisting catalog & contracts', { - executionContext: context.source, ->>>>>>> bf47229a2b (refactor(catalog): tune manager gating and sync log levels (#0)) catalogId: sourceCatalog.id, contractsCreationsCount: catalogSyncDiff.contractsCreations.length, contractsUpdatesCount: catalogSyncDiff.contractsUpdates.length, @@ -432,15 +432,15 @@ const cleanupObsoleteCatalogs = async (context: AuthContext, syncedCatalogs: str } }; -export const synchronizeCatalogs = async (context: AuthContext, user: AuthUser) => { +export const synchronizeCatalogs = async ( + context: AuthContext, + user: AuthUser, + options?: CatalogSyncSourceGatewayOptions, +) => { const sources = initSyncSources(); const filigranCatalogRemoteUri = getFiligranCatalogRemoteUri(); -<<<<<<< HEAD logApp.debug('[OPENCTI-MODULE] Synchronizing catalogs', { module: 'catalog', -======= - logCatalog.info('[OPENCTI-MODULE] Synchronizing catalogs', { ->>>>>>> bf47229a2b (refactor(catalog): tune manager gating and sync log levels (#0)) executionContext: context.source, count: sources.length, filigranCatalogRemoteUri, @@ -449,7 +449,7 @@ export const synchronizeCatalogs = async (context: AuthContext, user: AuthUser) const syncedCatalogsWithChanges: string[] = []; // Sync catalogs from sources for (const source of sources) { - let result = await synchronizeCatalog(context, user, source); + let result = await synchronizeCatalog(context, user, source, options); if (source.kind === 'remote' && result.error && filigranCatalogRemoteUri && source.uri === filigranCatalogRemoteUri) { const alreadyPersistedRemoteCatalog = await findCatalogBySourceUri(context, user, source.uri); if (!alreadyPersistedRemoteCatalog) { @@ -458,7 +458,7 @@ export const synchronizeCatalogs = async (context: AuthContext, user: AuthUser) sourceUri: source.uri, module: 'catalog', }); - result = await synchronizeCatalog(context, user, EMBEDDED_CATALOG_SYNC_SOURCE); + result = await synchronizeCatalog(context, user, EMBEDDED_CATALOG_SYNC_SOURCE, options); } } if (!result.error) { diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-source-gateway.ts b/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-source-gateway.ts index 5f1f6711ad3b..ac98e9758f44 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-source-gateway.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-source-gateway.ts @@ -1,5 +1,5 @@ import { readFile } from 'node:fs/promises'; -import conf, { logApp } from '../../../config/conf'; +import { logApp } from '../../../config/conf'; import { isEmptyField } from '../../../database/utils'; import { UnsupportedError } from '../../../config/errors'; import { getOrCompileValidator } from '../catalog-domain'; @@ -59,8 +59,12 @@ interface CatalogSyncSourceAdapter { } const DEFAULT_REMOTE_CATALOG_TIMEOUT_MS = 30000; -const resolveRemoteCatalogTimeoutMs = () => { - const configuredTimeout = Number(conf.get('app:catalog_sync_remote_timeout') ?? conf.get('app:request_timeout')); +export type CatalogSyncSourceGatewayOptions = { + remoteCatalogTimeoutMs?: number; +}; + +const resolveRemoteCatalogTimeoutMs = (options?: CatalogSyncSourceGatewayOptions) => { + const configuredTimeout = Number(options?.remoteCatalogTimeoutMs); if (!Number.isFinite(configuredTimeout) || configuredTimeout <= 0) { return DEFAULT_REMOTE_CATALOG_TIMEOUT_MS; } @@ -96,17 +100,20 @@ class LocalCatalogSyncSource implements CatalogSyncSourceAdapter { } class RemoteCatalogSyncSource implements CatalogSyncSourceAdapter { - constructor(private sourceConfig: Extract) {} + constructor( + private sourceConfig: Extract, + private options?: CatalogSyncSourceGatewayOptions, + ) {} async fetch() { - const timeout = resolveRemoteCatalogTimeoutMs(); + const timeout = resolveRemoteCatalogTimeoutMs(this.options); const client = getHttpClient({ responseType: 'text', timeout }); const response = await withAbortTimeout(timeout, (signal) => client.get(this.sourceConfig.uri, { signal })); return JSON.parse(response.data); } async fetchRevisionHint() { - const timeout = resolveRemoteCatalogTimeoutMs(); + const timeout = resolveRemoteCatalogTimeoutMs(this.options); const client = getHttpClient({ responseType: 'text', timeout }); const response = await withAbortTimeout(timeout, (signal) => client.head(this.sourceConfig.uri, { signal })); const etagHeader = response.headers?.etag; @@ -118,7 +125,10 @@ class RemoteCatalogSyncSource implements CatalogSyncSourceAdapter { } } -const getCatalogSourceAdapter = (config: CatalogSyncSourceConfig): CatalogSyncSourceAdapter => { +const getCatalogSourceAdapter = ( + config: CatalogSyncSourceConfig, + options?: CatalogSyncSourceGatewayOptions, +): CatalogSyncSourceAdapter => { switch (config.kind) { case 'local': { return new LocalCatalogSyncSource(config); @@ -127,7 +137,7 @@ const getCatalogSourceAdapter = (config: CatalogSyncSourceConfig): CatalogSyncSo return new EmbeddedCatalogSyncSource(config); } case 'remote': { - return new RemoteCatalogSyncSource(config); + return new RemoteCatalogSyncSource(config, options); } default: throw UnsupportedError(`Unknown catalog sync source kind (${JSON.stringify(config)})`); @@ -318,8 +328,11 @@ const mapCatalogDtoV0ToCatalogSyncSource = (catalog: CatalogDtoV0): CatalogSyncS }; }; -export const fetchSourceCatalog = async (sourceConfig: CatalogSyncSourceConfig): Promise => { - const adapter = getCatalogSourceAdapter(sourceConfig); +export const fetchSourceCatalog = async ( + sourceConfig: CatalogSyncSourceConfig, + options?: CatalogSyncSourceGatewayOptions, +): Promise => { + const adapter = getCatalogSourceAdapter(sourceConfig, options); const raw = await adapter.fetch(); const syncSource = mapCatalogDtoToCatalogSyncSource(raw); validateSyncSource(syncSource); @@ -336,8 +349,11 @@ export const fetchSourceCatalog = async (sourceConfig: CatalogSyncSourceConfig): return syncSource; }; -export const fetchSourceCatalogRevisionHint = async (sourceConfig: CatalogSyncSourceConfig): Promise => { - const adapter = getCatalogSourceAdapter(sourceConfig); +export const fetchSourceCatalogRevisionHint = async ( + sourceConfig: CatalogSyncSourceConfig, + options?: CatalogSyncSourceGatewayOptions, +): Promise => { + const adapter = getCatalogSourceAdapter(sourceConfig, options); const revisionHint = await adapter.fetchRevisionHint?.(); if (sourceConfig.kind === 'remote') { if (revisionHint) { diff --git a/opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-sync-domain-test.ts b/opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-sync-domain-test.ts index b25a16cbd3e7..2f16cbd6eedc 100644 --- a/opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-sync-domain-test.ts +++ b/opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-sync-domain-test.ts @@ -132,7 +132,7 @@ describe('catalog-sync-domain', () => { mockFetchSourceCatalog.mockResolvedValue(buildSourceCatalog('embedded-catalog')); const result = await synchronizeCatalogs({ source: 'test' } as any, { id: 'user-1' } as any); expect(result).toEqual(['embedded-catalog']); - expect(mockFetchSourceCatalog).toHaveBeenCalledWith({ kind: 'embedded', uri: 'embedded' }, { source: 'test' }); + expect(mockFetchSourceCatalog).toHaveBeenCalledWith({ kind: 'embedded', uri: 'embedded' }, undefined); expect(mockFindCatalogs).toHaveBeenCalled(); }); @@ -168,7 +168,7 @@ describe('catalog-sync-domain', () => { mockFetchSourceCatalog.mockResolvedValue(buildSourceCatalog('embedded-catalog')); await synchronizeCatalogs({ source: 'test' } as any, { id: 'user-1' } as any); expect(mockFetchSourceCatalog).toHaveBeenCalledTimes(1); - expect(mockFetchSourceCatalog).toHaveBeenCalledWith({ kind: 'embedded', uri: 'embedded' }, { source: 'test' }); + expect(mockFetchSourceCatalog).toHaveBeenCalledWith({ kind: 'embedded', uri: 'embedded' }, undefined); }); it('should use custom catalog manager endpoint instead of xtm hub source when decoupling feature is enabled', async () => { @@ -184,7 +184,7 @@ describe('catalog-sync-domain', () => { mockFetchSourceCatalog.mockResolvedValue(buildSourceCatalog('custom-catalog')); await synchronizeCatalogs({ source: 'test' } as any, { id: 'user-1' } as any); expect(mockFetchSourceCatalog).toHaveBeenCalledTimes(1); - expect(mockFetchSourceCatalog).toHaveBeenCalledWith({ kind: 'remote', uri: 'https://catalog.example.org/custom.json' }, { source: 'test' }); + expect(mockFetchSourceCatalog).toHaveBeenCalledWith({ kind: 'remote', uri: 'https://catalog.example.org/custom.json' }, undefined); }); it('should fallback to embedded when remote decoupling source fails before first persistence', async () => { diff --git a/opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-sync-source-gateway-test.ts b/opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-sync-source-gateway-test.ts index 6e277ef7b39c..786e69bb3572 100644 --- a/opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-sync-source-gateway-test.ts +++ b/opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-sync-source-gateway-test.ts @@ -15,6 +15,12 @@ vi.mock('../../../../src/config/conf', () => ({ default: { get: vi.fn(() => undefined), }, + logApp: { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + }, TEST_MODE: true, ENABLED_METRICS: false, booleanConf: vi.fn(() => false), From 1fd96c4b4cadb577fa88cac51f53f65401464309 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Wed, 19 Aug 2026 21:11:06 +0200 Subject: [PATCH 23/45] fix: tests --- .../catalog-domain/ajvValidatorCache-test.ts | 15 ++------------- .../tests/01-unit/manager/telemetryTags-test.ts | 6 +++--- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/opencti-platform/opencti-graphql/tests/01-unit/domain/catalog-domain/ajvValidatorCache-test.ts b/opencti-platform/opencti-graphql/tests/01-unit/domain/catalog-domain/ajvValidatorCache-test.ts index 93f93e72296c..d85ec9890deb 100644 --- a/opencti-platform/opencti-graphql/tests/01-unit/domain/catalog-domain/ajvValidatorCache-test.ts +++ b/opencti-platform/opencti-graphql/tests/01-unit/domain/catalog-domain/ajvValidatorCache-test.ts @@ -1,13 +1,10 @@ -import { fileURLToPath } from 'node:url'; import Ajv from 'ajv'; -import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import type { CatalogContract } from '../../../../src/modules/catalog/catalog-types'; import type { ConnectorContractConfiguration } from '../../../../src/generated/graphql'; const compileSpy = vi.spyOn(Ajv.prototype, 'compile'); -const TEST_CATALOG_URL = new URL('../../../utils/opencti-manifest.json', import.meta.url); - let catalogDomain: typeof import('../../../../src/modules/catalog/catalog-domain'); const buildContract = (overrides: Partial = {}): CatalogContract => ({ @@ -33,17 +30,9 @@ const buildContract = (overrides: Partial = {}): CatalogContrac const buildConfig = (entries: Record): ConnectorContractConfiguration[] => Object.entries(entries).map(([key, value]) => ({ key, value })); describe('catalog-domain - AJV validator compilation cache', () => { - beforeAll(async () => { - process.env.APP__CUSTOM_CATALOGS = JSON.stringify([fileURLToPath(TEST_CATALOG_URL)]); + beforeEach(async () => { vi.resetModules(); catalogDomain = await import('../../../../src/modules/catalog/catalog-domain'); - }); - - afterAll(() => { - delete process.env.APP__CUSTOM_CATALOGS; - }); - - beforeEach(() => { compileSpy.mockClear(); }); diff --git a/opencti-platform/opencti-graphql/tests/01-unit/manager/telemetryTags-test.ts b/opencti-platform/opencti-graphql/tests/01-unit/manager/telemetryTags-test.ts index 0e29dd73a3c0..fa91797184ba 100644 --- a/opencti-platform/opencti-graphql/tests/01-unit/manager/telemetryTags-test.ts +++ b/opencti-platform/opencti-graphql/tests/01-unit/manager/telemetryTags-test.ts @@ -68,9 +68,9 @@ describe('Active connectors by catalog identity', () => { const managed = (image: string, type = 'EXTERNAL_IMPORT'): ConnectorIdentitySource => ({ catalog_id: 'catalog-1', manager_contract_image: image, - manager_contract: { - slug: CONTRACTS.get(image)!.slug, - } as CatalogContractEntityFields, + manager_contract: CONTRACTS.has(image) ? { + slug: CONTRACTS.get(image)?.slug, + } as CatalogContractEntityFields : undefined, name: 'User Renamed Me', connector_type: type, }); From ad552b8bb6c7d63488fef9899c3b4a239e3b40a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Wed, 19 Aug 2026 21:13:25 +0200 Subject: [PATCH 24/45] chore: default config changes --- .../opencti-graphql/config/default.json | 34 ++++++++++++++++--- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/opencti-platform/opencti-graphql/config/default.json b/opencti-platform/opencti-graphql/config/default.json index ad7032cd39fa..126878c4db9d 100644 --- a/opencti-platform/opencti-graphql/config/default.json +++ b/opencti-platform/opencti-graphql/config/default.json @@ -43,8 +43,13 @@ "logs_console": true, "logs_max_files": 7, "logs_directory": "./logs", - "logs_in_transports": ["administration"], - "trace_request_headers": ["user-agent", "x-forwarded-for"] + "logs_in_transports": [ + "administration" + ], + "trace_request_headers": [ + "user-agent", + "x-forwarded-for" + ] }, "event_loop_logs": { "enabled": false, @@ -180,11 +185,19 @@ }, "groups": { "enabled": true, - "protected_names": ["Administrators", "Connectors", "Default"] + "protected_names": [ + "Administrators", + "Connectors", + "Default" + ] }, "roles": { "enabled": true, - "protected_names": ["Administrator", "Connector", "Default"] + "protected_names": [ + "Administrator", + "Connector", + "Default" + ] }, "rules": { "enabled": true @@ -373,6 +386,13 @@ "lock_key": "exclusion_list_cache_sync_manager_lock", "interval": 10000 }, + "catalog_manager": { + "enabled": true, + "lock_key": "catalog_manager_lock", + "interval": 60000, + "custom_catalog_refresh_endpoint_uri": null, + "catalog_sync_remote_timeout": 30000 + }, "redis": { "mode": "single", "namespace": "", @@ -391,6 +411,8 @@ "engine_selector": "auto", "engine_check": true, "index_creation_pattern": "-000001", + "search_wildcard_prefix": false, + "search_fuzzy": false, "max_pagination_result": 5000, "default_pagination_result": 500, "max_bulk_operations": 5000, @@ -406,7 +428,9 @@ "access_key": "ChangeMe", "secret_key": "ChangeMe", "use_aws_role": false, - "excluded_files": [".DS_Store"], + "excluded_files": [ + ".DS_Store" + ], "disable_checksum_validation": false }, "rabbitmq": { From b90c7978fa21f6904810557022857f60ceefb2bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Wed, 19 Aug 2026 21:43:53 +0200 Subject: [PATCH 25/45] fix: test and keep stripping runtime keys --- .../src/database/repository.js | 2 +- .../02-resolvers/connector-composer-test.ts | 42 ++++++++++++++----- .../tests/utils/opencti-manifest.json | 8 ++-- 3 files changed, 36 insertions(+), 16 deletions(-) diff --git a/opencti-platform/opencti-graphql/src/database/repository.js b/opencti-platform/opencti-graphql/src/database/repository.js index f52a749fe057..f30a7853ed3c 100644 --- a/opencti-platform/opencti-graphql/src/database/repository.js +++ b/opencti-platform/opencti-graphql/src/database/repository.js @@ -79,7 +79,7 @@ export const computeManagerConnectorContract = async (_context, _user, cn) => { if (!cn.manager_contract) { return null; } - return JSON.stringify(mapContractEntityFieldsToGraphqlCatalogContract(cn.manager_contract)); + return JSON.stringify(mapContractEntityFieldsToGraphqlCatalogContract(cn.manager_contract, { excludeRuntimeConfigVars: true })); }; export const computeManagerConnectorExcerpt = async (_context, _user, cn) => { diff --git a/opencti-platform/opencti-graphql/tests/03-integration/02-resolvers/connector-composer-test.ts b/opencti-platform/opencti-graphql/tests/03-integration/02-resolvers/connector-composer-test.ts index e272d4229f9b..db95ba91e373 100644 --- a/opencti-platform/opencti-graphql/tests/03-integration/02-resolvers/connector-composer-test.ts +++ b/opencti-platform/opencti-graphql/tests/03-integration/02-resolvers/connector-composer-test.ts @@ -1,13 +1,15 @@ import { expect, it, describe, afterAll, beforeAll, vi } from 'vitest'; import gql from 'graphql-tag'; import { v4 as uuidv4 } from 'uuid'; -import { fileURLToPath } from 'node:url'; +import { FEATURE_FLAG_ALL, ENABLED_FEATURE_FLAGS } from '../../../src/config/conf'; import { queryAsAdminWithError, awaitUntilCondition, queryAsAdminWithSuccess, queryAsUserIsExpectedForbidden, queryAsUserWithSuccess } from '../../utils/testQueryHelper'; -import { USER_CONNECTOR, USER_EDITOR } from '../../utils/testQuery'; +import { ADMIN_USER, USER_CONNECTOR, USER_EDITOR, testContext } from '../../utils/testQuery'; import { wait } from '../../../src/database/utils'; import { XTMComposerMock } from '../../utils/XTMComposerMock'; import type { ApiConnector } from '../../utils/XTMComposerMock'; import { catalogHelper } from '../../utils/catalogHelper'; +import { synchronizeCatalogs } from '../../../src/modules/catalog/sync/catalog-sync-domain'; +import conf from '../../../src/config/conf'; const TEST_COMPOSER_ID = uuidv4(); const TEST_USER_CONNECTOR_ID: string = USER_CONNECTOR.id; // Initialize with default value @@ -250,12 +252,21 @@ describe('Connector Composer and Managed Connectors', () => { // Track all created resources const createdConnectorIds = new Set(); let xtmComposer: XTMComposerMock; + const previousEnabledFeatureFlags = [...ENABLED_FEATURE_FLAGS]; + const previousCustomCatalogRefreshEndpoint = conf.get('catalog_manager:custom_catalog_refresh_endpoint_uri'); + const previousCustomCatalogs = conf.get('app:custom_catalogs'); // Initialize XTM Composer mock beforeAll(async () => { - // Set up test catalog path in environment - const testCatalogUrl = new URL('../../utils/opencti-manifest.json', import.meta.url); - process.env.APP__CUSTOM_CATALOGS = JSON.stringify([fileURLToPath(testCatalogUrl)]); + const testCatalogUri = new URL('../../utils/opencti-manifest.json', import.meta.url).toString(); + ENABLED_FEATURE_FLAGS.splice(0, ENABLED_FEATURE_FLAGS.length); + ENABLED_FEATURE_FLAGS.push(...previousEnabledFeatureFlags.filter((flag) => flag !== FEATURE_FLAG_ALL)); + if (!ENABLED_FEATURE_FLAGS.includes('DECOUPLING_VERSIONS')) { + ENABLED_FEATURE_FLAGS.push('DECOUPLING_VERSIONS'); + } + conf.set('app:custom_catalogs', []); + conf.set('catalog_manager:custom_catalog_refresh_endpoint_uri', testCatalogUri); + await synchronizeCatalogs(testContext, ADMIN_USER); // Validate that we're using the test catalog catalogHelper.validateTestCatalog(); @@ -1346,7 +1357,6 @@ describe('Connector Composer and Managed Connectors', () => { slug: expect.any(String), description: expect.any(String), short_description: expect.any(String), - logo: expect.any(String), use_cases: expect.any(Array), verified: expect.any(Boolean), last_verified_date: expect.any(String), @@ -1360,6 +1370,7 @@ describe('Connector Composer and Managed Connectors', () => { container_image: expect.any(String), container_type: expect.any(String), }); + expect(definition.logo === null || typeof definition.logo === 'string').toBeTruthy(); expect(definition.config_schema).toBeDefined(); expect(definition.config_schema.properties).toBeDefined(); const runtimeKeys = ['OPENCTI_TOKEN', 'OPENCTI_URL', 'CONNECTOR_TYPE', 'CONNECTOR_RUN_AND_TERMINATE']; @@ -1376,11 +1387,12 @@ describe('Connector Composer and Managed Connectors', () => { expect(excerpt).toMatchObject({ title: expect.any(String), slug: expect.any(String), - logo: expect.any(String), + logo: expect.anything(), }); }); it('should retrieve the manager contract excerpt for a managed connector', async () => { + const testConnector = catalogHelper.getTestSafeConnector(); const result = await queryAsAdminWithSuccess({ query: GET_CONNECTOR_EXCERPT_QUERY, variables: { id: managedConnectorId }, @@ -1389,9 +1401,9 @@ describe('Connector Composer and Managed Connectors', () => { expect(result.data).toBeDefined(); const excerpt = result.data?.connector?.manager_contract_excerpt; expect(excerpt).toBeDefined(); - expect(excerpt.title).toEqual('IPinfo'); - expect(excerpt.slug).toEqual('ipinfo'); - expect(excerpt.logo).toMatch(/^\/storage\/view\/catalog-logos\//); + expect(excerpt.title).toEqual(testConnector.title); + expect(excerpt.slug).toEqual(testConnector.slug); + expect(excerpt.logo === null || typeof excerpt.logo === 'string').toBeTruthy(); }); it('should return connectorMigrationAssessment with stable output format', async () => { @@ -1489,12 +1501,16 @@ describe('Connector Composer and Managed Connectors', () => { expect(result.data).toBeDefined(); expect(result.data?.managedConnectorEdit.name).toEqual('Updated IpInfo Connector'); expect(result.data?.managedConnectorEdit.manager_contract_configuration).toBeDefined(); - const runtimeKeys = ['OPENCTI_TOKEN', 'OPENCTI_URL', 'CONNECTOR_TYPE', 'CONNECTOR_RUN_AND_TERMINATE']; + const runtimeKeys = ['OPENCTI_URL', 'CONNECTOR_RUN_AND_TERMINATE']; runtimeKeys.forEach((runtimeKey) => { const found = result.data?.managedConnectorEdit.manager_contract_configuration .find((c: any) => c.key === runtimeKey); expect(found).toBeUndefined(); }); + const openctiTokenConfig = result.data?.managedConnectorEdit.manager_contract_configuration + .find((c: any) => c.key === 'OPENCTI_TOKEN'); + expect(openctiTokenConfig).toBeDefined(); + expect(openctiTokenConfig.value).not.toEqual('updated-token-456'); const autoConfig = result.data?.managedConnectorEdit.manager_contract_configuration .find((c: any) => c.key === 'CONNECTOR_AUTO'); expect(autoConfig.value).toEqual('false'); @@ -1839,5 +1855,9 @@ describe('Connector Composer and Managed Connectors', () => { } }), ); + + conf.set('catalog_manager:custom_catalog_refresh_endpoint_uri', previousCustomCatalogRefreshEndpoint); + conf.set('app:custom_catalogs', previousCustomCatalogs); + ENABLED_FEATURE_FLAGS.splice(0, ENABLED_FEATURE_FLAGS.length, ...previousEnabledFeatureFlags); }); }); diff --git a/opencti-platform/opencti-graphql/tests/utils/opencti-manifest.json b/opencti-platform/opencti-graphql/tests/utils/opencti-manifest.json index b233b6c9e570..b541cc28f9e5 100644 --- a/opencti-platform/opencti-graphql/tests/utils/opencti-manifest.json +++ b/opencti-platform/opencti-graphql/tests/utils/opencti-manifest.json @@ -9,7 +9,7 @@ "slug": "test-external", "description": "This is a dummy external connector for testing purposes. It simulates an external data import connector with various configuration options.", "short_description": "A test external connector for importing dummy threat intelligence data", - "logo": "https://example.com/logos/test-external.png", + "logo": null, "use_cases": [ "Test Import", "Dummy Data" @@ -120,7 +120,7 @@ "slug": "test-email", "description": "A dummy email connector that simulates importing threat intelligence from email sources. This connector demonstrates complex configuration with arrays and optional fields.", "short_description": "Test connector for email-based threat intelligence import", - "logo": "https://example.com/logos/test-email.png", + "logo": null, "use_cases": [ "Email Import", "Test Processing" @@ -246,7 +246,7 @@ "slug": "test-enrichment", "description": "Dummy enrichment connector for testing IP address and domain enrichment functionality", "short_description": "Test IP and domain enrichment service", - "logo": "https://example.com/logos/test-enrichment.png", + "logo": null, "use_cases": [ "Test Enrichment", "IP Analysis" @@ -361,7 +361,7 @@ "slug": "test-stream", "description": "A dummy stream connector that simulates exporting OpenCTI data to external systems. Used for testing stream connector functionality.", "short_description": "Test connector for streaming data to external systems", - "logo": "https://example.com/logos/test-stream.png", + "logo": null, "use_cases": [ "Data Export", "Stream Testing" From 49d58de008e6c84d5dde2397465cffc60393aafd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Wed, 19 Aug 2026 21:46:41 +0200 Subject: [PATCH 26/45] fix: apply PR feedback Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../src/modules/catalog/catalog-http.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-http.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-http.ts index 0e188ced3820..546f7ec63cc0 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-http.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-http.ts @@ -18,11 +18,11 @@ export const handleCatalogLogoViewRequest: RequestHandler = async (req, res) => return; } const fileParam = req.params.file; - const file = Array.isArray(fileParam) ? fileParam[0] : fileParam; - logApp.debug('Catalog logo view handler', { file }); - if (typeof file !== 'string') { - throw UnsupportedError('Invalid URL format'); - } +const file = Array.isArray(fileParam) ? fileParam[0] : fileParam; +logApp.debug('Catalog logo view handler', { file }); +if (typeof file !== 'string' || file.includes('..') || file.includes('/') || file.includes('\\')) { + throw UnsupportedError('Invalid URL format'); +} const s3Key = `${CATALOG_CONTRACT_LOGOS_DIR}/${file}`; const stream = await downloadFile(s3Key); if (!stream) { From ed84c045cccb74fb47916027f76c001f5dd897bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Wed, 19 Aug 2026 21:53:13 +0200 Subject: [PATCH 27/45] fix: apply PR comments --- .../src/modules/connector/connector-domain.ts | 32 ++++++++++++------- .../02-resolvers/connector-migration-test.ts | 1 - 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/opencti-platform/opencti-graphql/src/modules/connector/connector-domain.ts b/opencti-platform/opencti-graphql/src/modules/connector/connector-domain.ts index 2c779690f7fd..b9d24fd93bcf 100644 --- a/opencti-platform/opencti-graphql/src/modules/connector/connector-domain.ts +++ b/opencti-platform/opencti-graphql/src/modules/connector/connector-domain.ts @@ -63,14 +63,18 @@ const autoUpgradeManagedConnector = async ( // Unsure how correct this is. Maybe the context_data is too big here. void publishUserAction({ event_type: 'mutation', - event_access: 'extended', + event_access: 'administration', event_scope: 'update', - message: 'Upgraded connector to latest compatible version', + message: 'upgrades connector to latest compatible version', user, context_data: { entity_type: ENTITY_TYPE_CONNECTOR, id: managedConnector.id, - input: patch, + input: { + slug, + previousVersion: contract_version, + newVersion: latestCompatibleContract.contract_version, + }, }, }); } else if (semver.gt(contract_version, latestCompatibleContract.contract_version)) { @@ -82,17 +86,20 @@ const autoUpgradeManagedConnector = async ( newVersion: latestCompatibleContract.contract_version, }); // Activity log - // Unsure how correct this is. Maybe the context_data is too big here. void publishUserAction({ event_type: 'mutation', - event_access: 'extended', + event_access: 'administration', event_scope: 'update', - message: 'Downgraded connector to latest compatible version', + message: 'downgrades connector to latest compatible version', user, context_data: { entity_type: ENTITY_TYPE_CONNECTOR, id: managedConnector.id, - input: patch, + input: { + slug, + previousVersion: contract_version, + newVersion: latestCompatibleContract.contract_version, + }, }, }); } else if (semver.eq(contract_version, latestCompatibleContract.contract_version)) { @@ -103,17 +110,20 @@ const autoUpgradeManagedConnector = async ( contractVersion: contract_version, }); // Activity log - // Unsure how correct this is. Maybe the context_data is too big here. void publishUserAction({ event_type: 'mutation', - event_access: 'extended', + event_access: 'administration', event_scope: 'update', - message: 'Upgraded connector to latest compatible identical version', + message: 'upgrades connector to latest compatible identical version', user, context_data: { entity_type: ENTITY_TYPE_CONNECTOR, id: managedConnector.id, - input: patch, + input: { + slug, + previousVersion: contract_version, + newVersion: latestCompatibleContract.contract_version, + }, }, }); } else { diff --git a/opencti-platform/opencti-graphql/tests/03-integration/02-resolvers/connector-migration-test.ts b/opencti-platform/opencti-graphql/tests/03-integration/02-resolvers/connector-migration-test.ts index c6e5bcb022c7..945f63563922 100644 --- a/opencti-platform/opencti-graphql/tests/03-integration/02-resolvers/connector-migration-test.ts +++ b/opencti-platform/opencti-graphql/tests/03-integration/02-resolvers/connector-migration-test.ts @@ -197,7 +197,6 @@ describe('Check connector migration', () => { const RUNTIME_KEYS = ['OPENCTI_TOKEN', 'CONNECTOR_ID', 'CONNECTOR_TYPE', 'HTTP_PROXY', 'HTTPS_PROXY', 'NO_PROXY', 'HTTPS_PROXY_REJECT_UNAUTHORIZED']; const managedConnector = managedConnectorResult.data.connectorMigrateToManaged; const rawConfig = managedConnector.manager_contract_configuration; - rawConfig.filter((c: any) => !RUNTIME_KEYS.includes(c.key)); const actualConfig = rawConfig.filter((c: { key: string }) => !RUNTIME_KEYS.includes(c.key)); RUNTIME_KEYS.forEach((runtimeKey) => { From 1e7964c2dcbcb3258a4c95d8ca5e7011a5bce9c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Wed, 19 Aug 2026 22:06:32 +0200 Subject: [PATCH 28/45] fix: remove leftover test --- .../tests/01-unit/utils/logger-test.ts | 164 ------------------ 1 file changed, 164 deletions(-) delete mode 100644 opencti-platform/opencti-graphql/tests/01-unit/utils/logger-test.ts diff --git a/opencti-platform/opencti-graphql/tests/01-unit/utils/logger-test.ts b/opencti-platform/opencti-graphql/tests/01-unit/utils/logger-test.ts deleted file mode 100644 index b85acde83b86..000000000000 --- a/opencti-platform/opencti-graphql/tests/01-unit/utils/logger-test.ts +++ /dev/null @@ -1,164 +0,0 @@ -import * as R from 'ramda'; -import { describe, expect, it } from 'vitest'; -import { appLogLevelMaxArraySize, appLogLevelMaxStringSize, logApp, prepareLogMetadata } from '../../../src/config/conf'; -import { FunctionalError } from '../../../src/config/errors'; - -// region objects definition -const CLASSIC_OBJECT = { - category: 'APP', - errors: [ - { - attributes: { - errors: [ - { - index: 'opencti_stix_domain_objects-000001', - index_uuid: 'ntQ2slJaRmWphjOmcls3lA', - reason: 'reason', - shard: '0', - type: 'version_conflict_engine_exception' - } - ], - genre: 'TECHNICAL', - http_status: 500 - }, - message: 'Bulk indexing fail', - name: 'DATABASE_ERROR', - stack: 'GraphQLError: Bulk indexing fail' - } - ], - id: '3f001108-c42c-4131-b3a3-583a98043c15', - level: 'error', - manager: 'RETENTION_MANAGER', - message: 'Bulk indexing fail', - source: 'backend', - timestamp: '2025-01-09T20:57:05.422Z', - version: '6.4.6' -}; -const CATEGORY_TO_LIMIT = [ - 'Item 1', - 'Item 2', - 'Item 3', - 'Item 4', - 'Item 5', - 'Item 6', - 'Item 7', - 'Item 8', - 'Item 9', - 'Item 10', - 'Item 11', - 'Item 12', - 'Item 13', - 'Item 14', - 'Item 15', - 'Item 16', - 'Item 17', - 'Item 18', - 'Item 19', - 'Item 20', - 'Item 21', - 'Item 22', - 'Item 23', - 'Item 24', - 'Item 25', - 'Item 26', - 'Item 27', - 'Item 28', - 'Item 29', - 'Item 30', - 'Item 31', - 'Item 32', - 'Item 33', - 'Item 34', - 'Item 35', - 'Item 36', - 'Item 37', - 'Item 38', - 'Item 39', - 'Item 40', - 'Item 41', - 'Item 42', - 'Item 43', - 'Item 44', - 'Item 45', - 'Item 46', - 'Item 47', - 'Item 48', - 'Item 49', - 'Item 50', - 'Item 51', - 'Item 52', - 'Item 53', - 'Item 54', - 'Item 55', - 'Item 56' -]; -const TOO_COMPLEX_OBJECT = { - category: 'APP', - cause: FunctionalError('my error', { category_to_limit: CATEGORY_TO_LIMIT }), - errors: [ - { - category_to_limit: CATEGORY_TO_LIMIT - }, - { - category_to_limit: ['2', '1', '3'], - } - ], - id: '3f001108-c42c-4131-b3a3-583a98043c15', - level: 'error', - source: R.range(1, 6000).map(() => 'A').join(''), - timestamp: '2025-01-09T20:57:05.422Z' -}; -const WITH_ERROR_OBJECT = { - level: 'error', - cause: FunctionalError('my error', { cause: new Error('embedded error') }), - timestamp: '2025-01-09T20:57:05.422Z' -}; -// endregion - -describe('Logger test suite', () => { - it('should expose child logger helpers', () => { - const child = logApp.child({ module: 'catalog', source: 'backend' }); - expect(child).toBeDefined(); - expect(typeof child.info).toBe('function'); - expect(typeof child.warn).toBe('function'); - expect(typeof child.error).toBe('function'); - expect(typeof child.debug).toBe('function'); - }); - - it('Log object is correctly untouched', () => { - const cleanObject = prepareLogMetadata(CLASSIC_OBJECT); - const classicCompare = R.dissoc('version', CLASSIC_OBJECT); - const cleanCompare = R.dissoc('version', cleanObject); - expect(JSON.stringify(cleanCompare)).toEqual(JSON.stringify(classicCompare)); - }); - - it('Log object with error correctly formatted', () => { - const cleanObject = prepareLogMetadata(WITH_ERROR_OBJECT); - expect(cleanObject.cause.message).toBe('my error'); - expect(cleanObject.cause.attributes.cause.message).toBe('embedded error'); - }); - - it('Log object is correctly limited', () => { - let initialSize = CATEGORY_TO_LIMIT.length; - const start = new Date().getTime(); - const cleanObject = prepareLogMetadata(TOO_COMPLEX_OBJECT); - const parsingTimeMs = new Date().getTime() - start; - expect(parsingTimeMs).to.be.lt(8); - let cleanedSize = cleanObject.cause.attributes.category_to_limit.length; - expect(initialSize).not.toEqual(cleanedSize); - expect(initialSize).to.be.gt(appLogLevelMaxArraySize); - expect(cleanedSize).to.be.eq(appLogLevelMaxArraySize); - // check more inside look - initialSize = TOO_COMPLEX_OBJECT.errors[0].category_to_limit.length; - cleanedSize = cleanObject.errors[0].category_to_limit.length; - expect(initialSize).not.toEqual(cleanedSize); - expect(initialSize).to.be.gt(appLogLevelMaxArraySize); - expect(cleanedSize).to.be.eq(appLogLevelMaxArraySize); - initialSize = TOO_COMPLEX_OBJECT.source.length; - cleanedSize = cleanObject.source.length; - // Check max string - expect(initialSize).not.toEqual(cleanedSize); - expect(initialSize).to.be.gt(appLogLevelMaxStringSize); - expect(cleanedSize).to.be.eq(appLogLevelMaxStringSize); - }); -}); From 72e96dadd410ccabeabc3876826b2f61bfcf2039 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Wed, 19 Aug 2026 22:36:09 +0200 Subject: [PATCH 29/45] fix(catalog): restore string version compatibility and local ordering (#0) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../src/database/engine-mapping-generator.ts | 4 - .../src/modules/catalog/catalog-repository.ts | 101 ++---------------- .../modules/catalog/catalog-version-utils.ts | 76 +++++++++++++ .../src/modules/catalog/catalog.ts | 4 +- .../src/schema/attribute-definition.ts | 5 +- .../src/schema/schema-attributes.ts | 4 - .../opencti-graphql/src/utils/sorting.ts | 8 +- .../catalog/catalog-version-utils-test.ts | 93 ++++++++++++++++ 8 files changed, 185 insertions(+), 110 deletions(-) create mode 100644 opencti-platform/opencti-graphql/src/modules/catalog/catalog-version-utils.ts create mode 100644 opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-version-utils-test.ts diff --git a/opencti-platform/opencti-graphql/src/database/engine-mapping-generator.ts b/opencti-platform/opencti-graphql/src/database/engine-mapping-generator.ts index 07a8c2a2093a..3c2c745aba04 100644 --- a/opencti-platform/opencti-graphql/src/database/engine-mapping-generator.ts +++ b/opencti-platform/opencti-graphql/src/database/engine-mapping-generator.ts @@ -11,7 +11,6 @@ import { shortMapping, shortStringFormats, textMapping, - versionMapping, } from '../schema/attribute-definition'; import { ABSTRACT_STIX_REF_RELATIONSHIP } from '../schema/general'; import { INTERNAL_RELATIONSHIPS } from '../schema/internalRelationship'; @@ -27,9 +26,6 @@ export const engineMappingGenerator = (engine: ElkClient | OpenClient): Record { if (entityAttribute.type === 'string') { - if (entityAttribute.format === 'version') { - return versionMapping; - } if (shortStringFormats.includes(entityAttribute.format)) { return shortMapping; } diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts index ae301cc3aaf1..54c67cf379c6 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts @@ -2,7 +2,7 @@ import { logApp, PLATFORM_VERSION } from '../../config/conf'; import { elDeleteInstances, elIndex, elIndexElements, elLoadBy } from '../../database/engine'; import { fullEntitiesList } from '../../database/middleware-loader'; import { INDEX_INTERNAL_OBJECTS, READ_INDEX_INTERNAL_OBJECTS } from '../../database/utils'; -import { FilterMode, FilterOperator, OrderingMode } from '../../generated/graphql'; +import { FilterMode, FilterOperator } from '../../generated/graphql'; import type { BasicStoreBase } from '../../types/store'; import type { AuthContext, AuthUser } from '../../types/user'; import { @@ -15,6 +15,7 @@ import { type CatalogUpsert, ENTITY_TYPE_CATALOG, } from './catalog-types'; +import { filterAndSortLatestCompatibleContracts } from './catalog-version-utils'; /** * Catalog data accessors & mutators @@ -115,46 +116,20 @@ export const findLatestCompatibleCatalogContractsByCatalogId = async ( key: ['catalog_id'], values: [catalogId], }], - filterGroups: [ - { - mode: FilterMode.Or, - filters: [], - filterGroups: [ - { - mode: FilterMode.And, - filters: [{ - key: ['support_version'], - values: [PLATFORM_VERSION], - operator: FilterOperator.Lte, - }], - filterGroups: [], - }, - { - mode: FilterMode.And, - filters: [{ - key: ['support_version'], - values: ['EXISTS'], - operator: FilterOperator.NotEq, - }], - filterGroups: [], - }, - ], - }, - ], + filterGroups: [], mode: FilterMode.And, }, - orderBy: 'contract_version', - orderMode: OrderingMode.Desc, }, ); + const compatibleContracts = filterAndSortLatestCompatibleContracts(contracts); logApp.debug('[OPENCTI-MODULE] Loaded compatible catalog contracts', { module: 'catalog', catalogId, platformVersion: PLATFORM_VERSION, - compatibleContractsCount: contracts.length, + compatibleContractsCount: compatibleContracts.length, }); // Keep latest compatible version by slug (results are sorted by version desc). - return contracts.reduce((map, contract) => { + return compatibleContracts.reduce((map, contract) => { if (map.has(contract.slug)) { return map; } @@ -179,40 +154,12 @@ export const findLatestCompatibleCatalogContractBySlug = async ( key: ['slug'], values: [contractSlug], }], - filterGroups: [ - { - mode: FilterMode.Or, - filters: [], - filterGroups: [ - { - mode: FilterMode.And, - filters: [{ - key: ['support_version'], - values: [PLATFORM_VERSION], - operator: FilterOperator.Lte, - }], - filterGroups: [], - }, - { - mode: FilterMode.And, - filters: [{ - key: ['support_version'], - values: ['EXISTS'], - operator: FilterOperator.NotEq, - }], - filterGroups: [], - }, - ], - }, - ], + filterGroups: [], mode: FilterMode.And, }, - orderBy: 'contract_version', - orderMode: OrderingMode.Desc, - first: 1, }, ); - return contracts[0]; + return filterAndSortLatestCompatibleContracts(contracts)[0]; }; export const findLatestCompatibleCatalogContractByImageName = async ( @@ -231,40 +178,12 @@ export const findLatestCompatibleCatalogContractByImageName = async ( key: ['image'], values: [imageName], }], - filterGroups: [ - { - mode: FilterMode.Or, - filters: [], - filterGroups: [ - { - mode: FilterMode.And, - filters: [{ - key: ['support_version'], - values: [PLATFORM_VERSION], - operator: FilterOperator.Lte, - }], - filterGroups: [], - }, - { - mode: FilterMode.And, - filters: [{ - key: ['support_version'], - values: ['EXISTS'], - operator: FilterOperator.NotEq, - }], - filterGroups: [], - }, - ], - }, - ], + filterGroups: [], mode: FilterMode.And, }, - orderBy: 'contract_version', - orderMode: OrderingMode.Desc, - first: 1, }, ); - const selectedContract = contracts[0]; + const selectedContract = filterAndSortLatestCompatibleContracts(contracts)[0]; if (!selectedContract) { logApp.debug('[OPENCTI-MODULE] No compatible catalog contract found by image', { module: 'catalog', diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-version-utils.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-version-utils.ts new file mode 100644 index 000000000000..21d45965ad9d --- /dev/null +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-version-utils.ts @@ -0,0 +1,76 @@ +import semver from 'semver'; +import { logApp, PLATFORM_VERSION } from '../../config/conf'; +import type { BasicStoreEntityCatalogContract } from './catalog-types'; + +type SupportVersionContract = Pick; +type ContractVersionContract = Pick; + +type CompatibilityOptions = { + platformVersion?: string; + onUnparsableSupportVersion?: (args: { contractId: string; supportVersion: string; platformVersion: string }) => void; +}; + +export const parseCatalogSemver = (version: string | null | undefined) => { + if (!version) { + return null; + } + return semver.coerce(version); +}; + +export const isSupportVersionCompatible = ( + contract: SupportVersionContract, + options: CompatibilityOptions = {}, +) => { + if (!contract.support_version) { + return true; + } + const platformVersion = options.platformVersion ?? PLATFORM_VERSION; + const contractVersion = parseCatalogSemver(contract.support_version); + const parsedPlatformVersion = parseCatalogSemver(platformVersion); + if (!contractVersion || !parsedPlatformVersion) { + if (options.onUnparsableSupportVersion) { + options.onUnparsableSupportVersion({ + contractId: contract.contract_id, + supportVersion: contract.support_version, + platformVersion, + }); + } else { + logApp.warn('[OPENCTI-MODULE] Ignoring catalog contract with unparsable support version', { + module: 'catalog', + contractId: contract.contract_id, + supportVersion: contract.support_version, + platformVersion, + }); + } + return false; + } + return semver.lte(contractVersion, parsedPlatformVersion); +}; + +export const compareContractVersionDesc = ( + left: ContractVersionContract, + right: ContractVersionContract, +) => { + const leftVersion = parseCatalogSemver(left.contract_version); + const rightVersion = parseCatalogSemver(right.contract_version); + if (leftVersion && rightVersion) { + return semver.rcompare(leftVersion, rightVersion); + } + if (leftVersion) { + return -1; + } + if (rightVersion) { + return 1; + } + return right.contract_version.localeCompare(left.contract_version, undefined, { numeric: true, sensitivity: 'base' }); +}; + +export const filterAndSortLatestCompatibleContracts = ( + contracts: BasicStoreEntityCatalogContract[], + options: CompatibilityOptions = {}, +) => { + return contracts + .filter((contract) => isSupportVersionCompatible(contract, options)) + .sort(compareContractVersionDesc); +}; + diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog.ts index ae1dce8a20b8..d2012a419660 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog.ts @@ -25,11 +25,11 @@ export const CATALOG_CONTRACT_MAPPINGS: AttributeDefinition[] = [ { name: 'last_verified_date', label: 'Last verified date', type: 'string', format: 'short', mandatoryType: 'no', editDefault: false, multiple: false, upsert: false, isFilterable: false }, { name: 'playbook_supported', label: 'Playbook supported', type: 'boolean', mandatoryType: 'no', editDefault: false, multiple: false, upsert: false, isFilterable: true }, { name: 'max_confidence_level', label: 'Max confidence level', type: 'numeric', precision: 'integer', mandatoryType: 'no', editDefault: false, multiple: false, upsert: false, isFilterable: false }, - { name: 'support_version', label: 'Support version', type: 'string', format: 'version', mandatoryType: 'no', editDefault: false, multiple: false, upsert: false, isFilterable: false }, + { name: 'support_version', label: 'Support version', type: 'string', format: 'short', mandatoryType: 'no', editDefault: false, multiple: false, upsert: false, isFilterable: false }, { name: 'subscription_link', label: 'Subscription link', type: 'string', format: 'short', mandatoryType: 'no', editDefault: false, multiple: false, upsert: false, isFilterable: false }, { name: 'source_code', label: 'Source code', type: 'string', format: 'short', mandatoryType: 'no', editDefault: false, multiple: false, upsert: false, isFilterable: false }, { name: 'manager_supported', label: 'Manager supported', type: 'boolean', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: true }, - { name: 'contract_version', label: 'Contract version', type: 'string', format: 'version', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: false }, + { name: 'contract_version', label: 'Contract version', type: 'string', format: 'short', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: false }, { name: 'image', label: 'Image', type: 'string', format: 'short', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: true }, { name: 'connector_type', label: 'Connector type', type: 'string', format: 'short', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: true }, { name: 'config_schema', label: 'Configuration schema', type: 'object', format: 'raw', mandatoryType: 'external', editDefault: false, multiple: false, upsert: false, isFilterable: false }, diff --git a/opencti-platform/opencti-graphql/src/schema/attribute-definition.ts b/opencti-platform/opencti-graphql/src/schema/attribute-definition.ts index 2311ea33745b..e38a8e2b14b7 100644 --- a/opencti-platform/opencti-graphql/src/schema/attribute-definition.ts +++ b/opencti-platform/opencti-graphql/src/schema/attribute-definition.ts @@ -20,7 +20,6 @@ export const shortMapping = { }, }; export const textMapping = { type: 'text' }; -export const versionMapping = { type: 'version' }; export const dateMapping = { type: 'date' }; export const booleanMapping = { type: 'boolean' }; export const numericMapping = (precision: string) => ({ type: precision, coerce: false }); @@ -77,7 +76,7 @@ export type DateAttribute = { type: 'date' } & BasicDefinition; export type BooleanAttribute = { type: 'boolean' } & BasicDefinition; export type NumericAttribute = { type: 'numeric'; precision: 'integer' | 'long' | 'float'; scalable?: boolean } & BasicDefinition; export type IdAttribute = { type: 'string'; format: 'id'; entityTypes: string[]; attrRawIds?: GetRawIdsFn; representative?: RepresentativeFn } & BasicDefinition; -export type TextAttribute = { type: 'string'; format: 'short' | 'text' | 'version' } & BasicDefinition; +export type TextAttribute = { type: 'string'; format: 'short' | 'text' } & BasicDefinition; export type EnumAttribute = { type: 'string'; format: 'enum'; values: string[] } & BasicDefinition; export type VocabAttribute = { type: 'string'; format: 'vocabulary'; vocabularyCategory: string } & BasicDefinition; export type JsonAttribute = { type: 'string'; format: 'json'; attrRawIds?: GetRawIdsFn; representative?: RepresentativeFn; multiple: false; schemaDef?: Record } & BasicDefinition; @@ -94,7 +93,7 @@ export type ComplexAttributeWithMappings = NumericAttribute | DateAttribute | BooleanAttribute | StringAttribute | ComplexAttribute | RefAttribute; -export const shortStringFormats = ['id', 'short', 'enum', 'vocabulary', 'version']; +export const shortStringFormats = ['id', 'short', 'enum', 'vocabulary']; export const longStringFormats = ['text', 'json']; // -- GLOBAL -- diff --git a/opencti-platform/opencti-graphql/src/schema/schema-attributes.ts b/opencti-platform/opencti-graphql/src/schema/schema-attributes.ts index 71bc3e44242e..cfd5625d17a2 100644 --- a/opencti-platform/opencti-graphql/src/schema/schema-attributes.ts +++ b/opencti-platform/opencti-graphql/src/schema/schema-attributes.ts @@ -286,10 +286,6 @@ export const isNumericAttribute = (k: string): boolean => ( export const isDateNumericOrBooleanAttribute = (k: string): boolean => ( schemaAttributesDefinition.isSpecificTypeAttribute(k, 'date', 'numeric', 'boolean') ); -export const isVersionAttribute = (k: string): boolean => { - const attribute = schemaAttributesDefinition.getAttributeByName(k); - return attribute?.type === 'string' && attribute?.format === 'version'; -}; export const isObjectAttribute = (k: string): boolean => ( schemaAttributesDefinition.isSpecificTypeAttribute(k, 'object') ); diff --git a/opencti-platform/opencti-graphql/src/utils/sorting.ts b/opencti-platform/opencti-graphql/src/utils/sorting.ts index 824ee68fac72..6c5f4c1ea938 100644 --- a/opencti-platform/opencti-graphql/src/utils/sorting.ts +++ b/opencti-platform/opencti-graphql/src/utils/sorting.ts @@ -1,10 +1,9 @@ -import { isDateNumericOrBooleanAttribute, isVersionAttribute, schemaAttributesDefinition } from '../schema/schema-attributes'; +import { isDateNumericOrBooleanAttribute, schemaAttributesDefinition } from '../schema/schema-attributes'; import { FunctionalError, UnsupportedError } from '../config/errors'; import { getPirWithAccessCheck } from '../modules/pir/pir-checkPirAccess'; import type { AuthContext, AuthUser } from '../types/user'; const PIR_ORDERING_CRITERIA = ['pir_score', 'last_pir_score_date']; -const VERSION_ORDERING_CRITERIA = ['version', 'support_version']; export const buildElasticSortingForAttributeCriteria = async ( context: AuthContext, @@ -48,13 +47,10 @@ export const buildElasticSortingForAttributeCriteria = async ( // criteria not in schema, attempt keyword sorting as a last resort if (!definition) { - if (VERSION_ORDERING_CRITERIA.includes(orderCriteria)) { - return { [orderCriteria]: { order: orderMode, missing: '_last' } }; - } return { [`${orderCriteria}.keyword`]: { order: orderMode, missing: '_last' } }; } - if (isDateNumericOrBooleanAttribute(orderCriteria) || isVersionAttribute(orderCriteria) || VERSION_ORDERING_CRITERIA.includes(orderCriteria)) { + if (isDateNumericOrBooleanAttribute(orderCriteria)) { // sorting on null dates results to an error, one way to fix it is to use missing: 0 // see https://github.com/elastic/elasticsearch/issues/81960 return { [orderCriteria]: { order: orderMode, missing: definition.type === 'date' ? 0 : '_last' } }; diff --git a/opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-version-utils-test.ts b/opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-version-utils-test.ts new file mode 100644 index 000000000000..85a78dcb11b7 --- /dev/null +++ b/opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-version-utils-test.ts @@ -0,0 +1,93 @@ +import { describe, expect, it, vi } from 'vitest'; +import type { BasicStoreEntityCatalogContract } from '../../../../src/modules/catalog/catalog-types'; +import { + compareContractVersionDesc, + filterAndSortLatestCompatibleContracts, + isSupportVersionCompatible, + parseCatalogSemver, +} from '../../../../src/modules/catalog/catalog-version-utils'; + +const buildContract = (args: { + contract_id: string; + contract_version: string; + support_version?: string; +}) => { + return { + contract_id: args.contract_id, + contract_version: args.contract_version, + support_version: args.support_version, + } as unknown as BasicStoreEntityCatalogContract; +}; + +describe('catalog-version-utils', () => { + it('should coerce semantic versions from strings', () => { + expect(parseCatalogSemver('>= 6.5.2')?.version).toBe('6.5.2'); + expect(parseCatalogSemver('7.2.0')?.version).toBe('7.2.0'); + expect(parseCatalogSemver(undefined)).toBeNull(); + }); + + it('should evaluate support version compatibility', () => { + expect(isSupportVersionCompatible( + buildContract({ contract_id: 'c1', contract_version: '1.0.0' }), + { platformVersion: '7.2.0' }, + )).toBe(true); + + expect(isSupportVersionCompatible( + buildContract({ contract_id: 'c2', contract_version: '1.0.0', support_version: '7.1.0' }), + { platformVersion: '7.2.0' }, + )).toBe(true); + + expect(isSupportVersionCompatible( + buildContract({ contract_id: 'c3', contract_version: '1.0.0', support_version: '8.0.0' }), + { platformVersion: '7.2.0' }, + )).toBe(false); + }); + + it('should report unparsable support versions as incompatible', () => { + const onUnparsableSupportVersion = vi.fn(); + const isCompatible = isSupportVersionCompatible( + buildContract({ contract_id: 'c4', contract_version: '1.0.0', support_version: 'not-a-version' }), + { platformVersion: '7.2.0', onUnparsableSupportVersion }, + ); + expect(isCompatible).toBe(false); + expect(onUnparsableSupportVersion).toHaveBeenCalledWith({ + contractId: 'c4', + supportVersion: 'not-a-version', + platformVersion: '7.2.0', + }); + }); + + it('should compare contract versions in descending semantic order', () => { + const contracts = [ + buildContract({ contract_id: 'c1', contract_version: '1.2.0' }), + buildContract({ contract_id: 'c2', contract_version: '2.0.0' }), + buildContract({ contract_id: 'c3', contract_version: '1.10.0' }), + ]; + const sorted = contracts.sort(compareContractVersionDesc); + expect(sorted.map((contract) => contract.contract_id)).toEqual(['c2', 'c3', 'c1']); + }); + + it('should filter incompatible contracts and keep latest versions first', () => { + const onUnparsableSupportVersion = vi.fn(); + const contracts = [ + buildContract({ contract_id: 'latest-compatible', contract_version: '2.0.0', support_version: '7.0.0' }), + buildContract({ contract_id: 'older-compatible', contract_version: '1.5.0', support_version: '>= 6.5.2' }), + buildContract({ contract_id: 'no-support-version', contract_version: '1.0.0' }), + buildContract({ contract_id: 'too-new', contract_version: '3.0.0', support_version: '8.0.0' }), + buildContract({ contract_id: 'invalid-support-version', contract_version: '2.5.0', support_version: 'broken-version' }), + ]; + + const filtered = filterAndSortLatestCompatibleContracts(contracts, { + platformVersion: '7.2.0', + onUnparsableSupportVersion, + }); + + expect(filtered.map((contract) => contract.contract_id)).toEqual([ + 'latest-compatible', + 'older-compatible', + 'no-support-version', + ]); + expect(onUnparsableSupportVersion).toHaveBeenCalledTimes(1); + }); +}); + From adcfca3f79ff198256ae23071b2f9d85cffb8bce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Wed, 19 Aug 2026 22:40:32 +0200 Subject: [PATCH 30/45] fix: linting issues --- .../src/domain/filterKeysSchema.ts | 4 +- .../catalog/catalog-sync-domain-test.ts | 2 +- .../connector-domain-auto-upgrade-test.ts | 60 ++++++++++--------- 3 files changed, 33 insertions(+), 33 deletions(-) diff --git a/opencti-platform/opencti-graphql/src/domain/filterKeysSchema.ts b/opencti-platform/opencti-graphql/src/domain/filterKeysSchema.ts index 7188a6302567..0ce3416e6808 100644 --- a/opencti-platform/opencti-graphql/src/domain/filterKeysSchema.ts +++ b/opencti-platform/opencti-graphql/src/domain/filterKeysSchema.ts @@ -89,8 +89,6 @@ const buildFilterDefinitionFromAttributeDefinition = (attributeDefinition: Attri if ((attributeDefinition as StringAttribute).format === 'id') { type = 'id'; elementsForFilterValuesSearch = (attributeDefinition as IdAttribute).entityTypes; - } else if ((attributeDefinition as StringAttribute).format === 'version') { - type = 'version'; } else if ((attributeDefinition as StringAttribute).format === 'short') { type = 'string'; } else if ((attributeDefinition as StringAttribute).format === 'vocabulary') { @@ -102,7 +100,7 @@ const buildFilterDefinitionFromAttributeDefinition = (attributeDefinition: Attri type = 'enum'; elementsForFilterValuesSearch = (attributeDefinition as EnumAttribute).values; } else { - throw Error(`A string attribute definition format can be 'id', 'short', 'version', 'text' or 'json', but not ${attributeDefinition.format}`); + throw Error(`A string attribute definition format can be 'id', 'short', 'text' or 'json', but not ${attributeDefinition.format}`); } } // return the filter definition diff --git a/opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-sync-domain-test.ts b/opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-sync-domain-test.ts index 2f16cbd6eedc..3b71cb3b8249 100644 --- a/opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-sync-domain-test.ts +++ b/opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-sync-domain-test.ts @@ -29,7 +29,7 @@ const { mockUpsertCatalog: vi.fn(), mockDeleteCatalogs: vi.fn(), mockListCatalogContractLogos: vi.fn(), - mockConfGet: vi.fn((key: string) => { + mockConfGet: vi.fn((key: string): any => { if (key === 'redis:ca') return []; if (key === 'redis:use_ssl') return false; return undefined; diff --git a/opencti-platform/opencti-graphql/tests/01-unit/modules/connector/connector-domain-auto-upgrade-test.ts b/opencti-platform/opencti-graphql/tests/01-unit/modules/connector/connector-domain-auto-upgrade-test.ts index 8162344dd348..5891642f08be 100644 --- a/opencti-platform/opencti-graphql/tests/01-unit/modules/connector/connector-domain-auto-upgrade-test.ts +++ b/opencti-platform/opencti-graphql/tests/01-unit/modules/connector/connector-domain-auto-upgrade-test.ts @@ -46,35 +46,37 @@ vi.mock('../../../../src/config/conf', () => ({ import { autoUpgradeManagedConnectors } from '../../../../src/modules/connector/connector-domain'; -const buildManagedConnector = (overrides: Partial = {}): BasicStoreEntityConnector => ({ - id: 'connector-1', - internal_id: 'connector-1', - name: 'Managed connector', - title: 'Managed connector', - connector_type: 'EXTERNAL_IMPORT', - active: true, - auto: true, - built_in: false, - only_contextual: false, - connector_scope: [], - updated_at: '', - connector_user_id: 'user-1', - connector_info: {} as any, - playbook_compatible: false, - xtm_one_intent: null, - connector_state: '', - connector_state_reset: false, - connector_trigger_filters: '', - catalog_id: 'catalog-1', - manager_upgrade_strategy: 'latest', - manager_contract_image: 'opencti/connector-test:1.0.0', - manager_contract: { - slug: 'ipinfo', - contract_version: '1.0.0', - content_hash: 'hash-1', - } as any, - ...overrides, -}); +const buildManagedConnector = (overrides: Partial = {}): BasicStoreEntityConnector => { + const connector = { + id: 'connector-1', + internal_id: 'connector-1', + name: 'Managed connector', + connector_type: 'EXTERNAL_IMPORT', + active: true, + auto: true, + built_in: false, + only_contextual: false, + connector_scope: '*', + updated_at: new Date(), + connector_user_id: 'user-1', + connector_info: {} as any, + playbook_compatible: false, + xtm_one_intent: null, + connector_state: '', + connector_state_reset: false, + connector_trigger_filters: '', + catalog_id: 'catalog-1', + manager_upgrade_strategy: 'latest', + manager_contract_image: 'opencti/connector-test:1.0.0', + manager_contract: { + slug: 'ipinfo', + contract_version: '1.0.0', + content_hash: 'hash-1', + } as any, + ...overrides, + }; + return connector as BasicStoreEntityConnector; +}; const latestCompatibleContract = { slug: 'ipinfo', From 458f57a910c85851e817ed56c9e7952eda4692e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Thu, 20 Aug 2026 01:16:47 +0200 Subject: [PATCH 31/45] fix: test --- .../opencti-graphql/tests/01-unit/utils/sorting-test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opencti-platform/opencti-graphql/tests/01-unit/utils/sorting-test.ts b/opencti-platform/opencti-graphql/tests/01-unit/utils/sorting-test.ts index f73ff9655200..f3c3db3f6290 100644 --- a/opencti-platform/opencti-graphql/tests/01-unit/utils/sorting-test.ts +++ b/opencti-platform/opencti-graphql/tests/01-unit/utils/sorting-test.ts @@ -32,7 +32,7 @@ describe('Sorting utilities', () => { sorting = await buildElasticSortingForAttributeCriteria(testContext, SYSTEM_USER, 'support_version', 'asc'); expect(sorting).toEqual({ - support_version: { + 'support_version.keyword': { missing: '_last', order: 'asc', }, From 1e6b09d596a7d210d366997df27383ad72a8d3e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Thu, 20 Aug 2026 02:49:30 +0200 Subject: [PATCH 32/45] fix: linting --- .../src/modules/catalog/catalog-http.ts | 10 +++++----- .../src/modules/catalog/catalog-version-utils.ts | 1 - 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-http.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-http.ts index 546f7ec63cc0..595a63773e45 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-http.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-http.ts @@ -18,11 +18,11 @@ export const handleCatalogLogoViewRequest: RequestHandler = async (req, res) => return; } const fileParam = req.params.file; -const file = Array.isArray(fileParam) ? fileParam[0] : fileParam; -logApp.debug('Catalog logo view handler', { file }); -if (typeof file !== 'string' || file.includes('..') || file.includes('/') || file.includes('\\')) { - throw UnsupportedError('Invalid URL format'); -} + const file = Array.isArray(fileParam) ? fileParam[0] : fileParam; + logApp.debug('Catalog logo view handler', { file }); + if (typeof file !== 'string' || file.includes('..') || file.includes('/') || file.includes('\\')) { + throw UnsupportedError('Invalid URL format'); + } const s3Key = `${CATALOG_CONTRACT_LOGOS_DIR}/${file}`; const stream = await downloadFile(s3Key); if (!stream) { diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-version-utils.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-version-utils.ts index 21d45965ad9d..b613ef08ac80 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-version-utils.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-version-utils.ts @@ -73,4 +73,3 @@ export const filterAndSortLatestCompatibleContracts = ( .filter((contract) => isSupportVersionCompatible(contract, options)) .sort(compareContractVersionDesc); }; - From 5d68ee36025acb96c172cde33dbdd300e3008fae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Mon, 14 Sep 2026 10:08:48 +0200 Subject: [PATCH 33/45] fix(catalog): resolve contract and catalog indices before writes (#16059) Catalog contract updates/deletions and catalog upserts now resolve the entity's physical index before writing, instead of relying on the INDEX_INTERNAL_OBJECTS write alias, which can diverge from the document's actual index (e.g. after an ILM rollover). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../src/modules/catalog/catalog-repository.ts | 61 ++++++-- .../catalog/sync/catalog-sync-domain.ts | 2 +- .../catalog/catalog-repository-test.ts | 132 ++++++++++++++++++ 3 files changed, 182 insertions(+), 13 deletions(-) create mode 100644 opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-repository-test.ts diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts index 54c67cf379c6..1b19be965c82 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts @@ -1,9 +1,9 @@ import { logApp, PLATFORM_VERSION } from '../../config/conf'; +import { FunctionalError } from '../../config/errors'; import { elDeleteInstances, elIndex, elIndexElements, elLoadBy } from '../../database/engine'; -import { fullEntitiesList } from '../../database/middleware-loader'; +import { fullEntitiesList, internalFindByIdsMapped } from '../../database/middleware-loader'; import { INDEX_INTERNAL_OBJECTS, READ_INDEX_INTERNAL_OBJECTS } from '../../database/utils'; import { FilterMode, FilterOperator } from '../../generated/graphql'; -import type { BasicStoreBase } from '../../types/store'; import type { AuthContext, AuthUser } from '../../types/user'; import { type BasicStoreEntityCatalogContract, @@ -39,8 +39,16 @@ export const findCatalogBySourceUri = async ( return catalog; }; -export const upsertCatalog = async (_context: AuthContext, _user: AuthUser, update: CatalogUpsert) => { - await elIndex(INDEX_INTERNAL_OBJECTS, { +export const upsertCatalog = async ( + _context: AuthContext, + _user: AuthUser, + update: CatalogUpsert, + currentCatalog?: BasicStoreEntityCatalog, +) => { + // Reuse the existing document's physical index when updating so the write lands + // on the same index as the current document instead of the write alias, which + // could resolve to a different physical index (e.g. after an ILM rollover). + await elIndex(currentCatalog?._index ?? INDEX_INTERNAL_OBJECTS, { ...update, entity_type: ENTITY_TYPE_CATALOG, }); @@ -73,6 +81,29 @@ export const deleteCatalogs = async (context: AuthContext, catalogEntities: Basi * Catalog contracts data accessors & mutators */ +const resolveCatalogContractsByIds = async ( + context: AuthContext, + user: AuthUser, + ids: string[], +) => { + const contractsById = await internalFindByIdsMapped( + context, + user, + ids, + { + type: ENTITY_TYPE_CATALOG_CONTRACT, + indices: [READ_INDEX_INTERNAL_OBJECTS], + baseData: true, + mapWithAllIds: true, + }, + ); + const missingIds = ids.filter((id) => !contractsById[id]); + if (missingIds.length > 0) { + throw FunctionalError('Catalog contracts not found', { ids: missingIds }); + } + return ids.map((id) => contractsById[id]); +}; + export const findCatalogContractsByCatalogId = async ( context: AuthContext, user: AuthUser, @@ -227,10 +258,15 @@ export const updateCatalogContracts = async ( user: AuthUser, updates: CatalogContractUpdate[], ) => { + const existingContracts = await resolveCatalogContractsByIds( + context, + user, + updates.map((update) => update.internal_id), + ); // We can use bulk `index` as we provide the entire documents - const contractsToIndex = updates.map((update) => ({ + const contractsToIndex = updates.map((update, index) => ({ ...update, - _index: INDEX_INTERNAL_OBJECTS, + _index: existingContracts[index]._index, entity_type: ENTITY_TYPE_CATALOG_CONTRACT, })); if (updates.length > 0) { @@ -253,18 +289,19 @@ export const updateCatalogContracts = async ( export const deleteCatalogContracts = async ( context: AuthContext, - _user: AuthUser, + user: AuthUser, deletions: CatalogContractDeletion[], ) => { - const docs = deletions.map((deletion) => ({ - _index: INDEX_INTERNAL_OBJECTS, - _id: deletion.idToDelete, - } as BasicStoreBase)); + const contracts = await resolveCatalogContractsByIds( + context, + user, + deletions.map((deletion) => deletion.idToDelete), + ); if (deletions.length > 0) { logApp.debug('[OPENCTI-MODULE] Deleting catalog contracts', { module: 'catalog', count: deletions.length, }); } - await elDeleteInstances(context, docs); + await elDeleteInstances(context, contracts); }; diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts b/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts index 3d53446d0a1c..060d061d11da 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts @@ -342,7 +342,7 @@ const synchronizeCatalog = async ( await updateCatalogContracts(context, user, catalogSyncDiff.contractsUpdates); } await deleteCatalogContracts(context, user, catalogSyncDiff.contractsDeletions); - await upsertCatalog(context, user, catalogSyncDiff.catalogUpsert); + await upsertCatalog(context, user, catalogSyncDiff.catalogUpsert, currentCatalog); const isNotNil = (str: string | null | undefined): str is string => Boolean(str); let usedLogos = catalogSyncDiff.contractsCreations.map(({ logo_uri }) => logo_uri).filter(isNotNil); usedLogos = usedLogos.concat(catalogSyncDiff.contractsUpdates.map(({ logo_uri }) => logo_uri).filter(isNotNil)); diff --git a/opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-repository-test.ts b/opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-repository-test.ts new file mode 100644 index 000000000000..c34a1db23053 --- /dev/null +++ b/opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-repository-test.ts @@ -0,0 +1,132 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { elDeleteInstances, elIndex, elIndexElements } from '../../../../src/database/engine'; +import { internalFindByIdsMapped } from '../../../../src/database/middleware-loader'; +import { deleteCatalogContracts, updateCatalogContracts, upsertCatalog } from '../../../../src/modules/catalog/catalog-repository'; +import { + type BasicStoreEntityCatalog, + type BasicStoreEntityCatalogContract, + type CatalogContractUpdate, + type CatalogUpsert, + ENTITY_TYPE_CATALOG, + ENTITY_TYPE_CATALOG_CONTRACT, +} from '../../../../src/modules/catalog/catalog-types'; +import { INDEX_INTERNAL_OBJECTS, READ_INDEX_INTERNAL_OBJECTS } from '../../../../src/database/utils'; +import type { AuthContext, AuthUser } from '../../../../src/types/user'; + +vi.mock('../../../../src/database/engine', () => ({ + elDeleteInstances: vi.fn(), + elIndex: vi.fn(), + elIndexElements: vi.fn(), + elLoadBy: vi.fn(), +})); + +vi.mock('../../../../src/database/middleware-loader', () => ({ + fullEntitiesList: vi.fn(), + internalFindByIdsMapped: vi.fn(), +})); + +const context = {} as AuthContext; +const user = {} as AuthUser; +const contractId = 'catalog-contract--1'; +const physicalIndex = 'opencti_internal_objects-000042'; +const existingContract = { + id: contractId, + internal_id: contractId, + _id: contractId, + _index: physicalIndex, + entity_type: ENTITY_TYPE_CATALOG_CONTRACT, +} as BasicStoreEntityCatalogContract; + +describe('catalog repository', () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(internalFindByIdsMapped).mockResolvedValue({ + [contractId]: existingContract, + }); + }); + + it('should upsert a new catalog on the write alias when no existing catalog is provided', async () => { + const catalogUpsert = { + internal_id: 'catalog--1', + standard_id: 'catalog--standard', + catalog_id: 'catalog-1', + } as CatalogUpsert; + + await upsertCatalog(context, user, catalogUpsert); + + expect(elIndex).toHaveBeenCalledWith(INDEX_INTERNAL_OBJECTS, { + ...catalogUpsert, + entity_type: ENTITY_TYPE_CATALOG, + }); + }); + + it('should upsert an existing catalog in its existing physical index', async () => { + const catalogPhysicalIndex = 'opencti_internal_objects-000042'; + const currentCatalog = { + id: 'catalog--1', + internal_id: 'catalog--1', + _index: catalogPhysicalIndex, + } as BasicStoreEntityCatalog; + const catalogUpsert = { + internal_id: 'catalog--1', + standard_id: 'catalog--standard', + catalog_id: 'catalog-1', + } as CatalogUpsert; + + await upsertCatalog(context, user, catalogUpsert, currentCatalog); + + expect(elIndex).toHaveBeenCalledWith(catalogPhysicalIndex, { + ...catalogUpsert, + entity_type: ENTITY_TYPE_CATALOG, + }); + }); + + it('should update catalog contracts in their existing physical index', async () => { + const update = { + internal_id: contractId, + standard_id: 'catalog-contract--standard', + title: 'Updated contract', + } as CatalogContractUpdate; + + await updateCatalogContracts(context, user, [update]); + + expect(internalFindByIdsMapped).toHaveBeenCalledWith( + context, + user, + [contractId], + { + type: ENTITY_TYPE_CATALOG_CONTRACT, + indices: [READ_INDEX_INTERNAL_OBJECTS], + baseData: true, + mapWithAllIds: true, + }, + ); + expect(elIndexElements).toHaveBeenCalledWith( + context, + user, + ENTITY_TYPE_CATALOG_CONTRACT, + [{ + ...update, + _index: physicalIndex, + entity_type: ENTITY_TYPE_CATALOG_CONTRACT, + }], + ); + }); + + it('should delete catalog contracts from their existing physical index', async () => { + await deleteCatalogContracts(context, user, [{ idToDelete: contractId }]); + + expect(elDeleteInstances).toHaveBeenCalledWith(context, [existingContract]); + }); + + it('should reject writes when a catalog contract cannot be located', async () => { + vi.mocked(internalFindByIdsMapped).mockResolvedValue({}); + + await expect(deleteCatalogContracts( + context, + user, + [{ idToDelete: contractId }], + )).rejects.toThrow('Catalog contracts not found'); + expect(elDeleteInstances).not.toHaveBeenCalled(); + }); +}); From 5a8ba4870430411ae11132b340c483859314f4a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Mon, 14 Sep 2026 10:13:37 +0200 Subject: [PATCH 34/45] refactor(catalog): clarify catalog lookup functions (#16059) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../src/modules/catalog/catalog-domain.ts | 4 +- .../src/modules/catalog/catalog-repository.ts | 34 ++++++++---- .../catalog/sync/catalog-sync-domain.ts | 4 +- .../catalog/catalog-repository-test.ts | 52 ++++++++++++++++++- .../catalog/catalog-sync-domain-test.ts | 12 ++--- 5 files changed, 83 insertions(+), 23 deletions(-) diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-domain.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-domain.ts index 5698cddf09cd..616491f1fa91 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-domain.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-domain.ts @@ -14,7 +14,7 @@ import { isEmptyField } from '../../database/utils'; import { UnsupportedError } from '../../config/errors'; import type { ConnectorContractConfiguration, ContractConfigInput } from '../../generated/graphql'; import type { ValidateFunction } from 'ajv'; -import { findCatalogByCatalogId, findCatalogs, findLatestCompatibleCatalogContractBySlug, findLatestCompatibleCatalogContractsByCatalogId } from './catalog-repository'; +import { findAllCatalogs, findCatalogByCatalogId, findLatestCompatibleCatalogContractBySlug, findLatestCompatibleCatalogContractsByCatalogId } from './catalog-repository'; import { logApp } from '../../config/conf'; const validatorCache = new Map(); @@ -452,7 +452,7 @@ export const queryCatalogById = async (context: AuthContext, user: AuthUser, cat }; export const queryCatalogs = async (context: AuthContext, user: AuthUser) => { - const catalogs = await findCatalogs(context, user); + const catalogs = await findAllCatalogs(context, user); const contracts = await Promise.all(catalogs.map((catalog) => findLatestCompatibleCatalogContractsByCatalogId(context, user, catalog.catalog_id))); const contractsTotalCount = contracts.reduce((total, contractsByCatalog) => total + contractsByCatalog.size, 0); logApp.debug('[OPENCTI-MODULE] Catalogs query resolved', { diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts index 1b19be965c82..788590a61889 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts @@ -1,7 +1,8 @@ +import type { FilterGroup } from '@mistralai/mistralai/models/components'; import { logApp, PLATFORM_VERSION } from '../../config/conf'; import { FunctionalError } from '../../config/errors'; import { elDeleteInstances, elIndex, elIndexElements, elLoadBy } from '../../database/engine'; -import { fullEntitiesList, internalFindByIdsMapped } from '../../database/middleware-loader'; +import { fullEntitiesList, internalFindByIdsMapped, type FilterGroupWithNested } from '../../database/middleware-loader'; import { INDEX_INTERNAL_OBJECTS, READ_INDEX_INTERNAL_OBJECTS } from '../../database/utils'; import { FilterMode, FilterOperator } from '../../generated/graphql'; import type { AuthContext, AuthUser } from '../../types/user'; @@ -54,8 +55,25 @@ export const upsertCatalog = async ( }); }; -export const findCatalogs = async (context: AuthContext, user: AuthUser, excludedIds?: string[]) => { - const filters = excludedIds?.length ? { +export const findAllCatalogs = async ( + context: AuthContext, + user: AuthUser, + filters?: FilterGroupWithNested, +) => { + return fullEntitiesList( + context, + user, + [ENTITY_TYPE_CATALOG], + { indices: [READ_INDEX_INTERNAL_OBJECTS], filters }, + ); +}; + +export const findAllCatalogsExcluding = async ( + context: AuthContext, + user: AuthUser, + excludedIds: string[], +) => { + const filters: FilterGroupWithNested = { filters: excludedIds.map((catalogId) => ({ key: ['catalog_id'], values: [catalogId], @@ -63,14 +81,8 @@ export const findCatalogs = async (context: AuthContext, user: AuthUser, exclude })), filterGroups: [], mode: FilterMode.And, - } : null; - const catalogs = await fullEntitiesList( - context, - user, - [ENTITY_TYPE_CATALOG], - { indices: [READ_INDEX_INTERNAL_OBJECTS], filters }, - ); - return catalogs; + }; + return findAllCatalogs(context, user, filters); }; export const deleteCatalogs = async (context: AuthContext, catalogEntities: BasicStoreEntityCatalog[]) => { diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts b/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts index 060d061d11da..abd8f80b70de 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/sync/catalog-sync-domain.ts @@ -26,7 +26,7 @@ import { findCatalogContractsByCatalogId, findCatalogByCatalogId, findCatalogBySourceUri, - findCatalogs, + findAllCatalogsExcluding, insertCatalogContracts, updateCatalogContracts, upsertCatalog, @@ -421,7 +421,7 @@ const initSyncSources = () => { }; const cleanupObsoleteCatalogs = async (context: AuthContext, syncedCatalogs: string[]) => { - const obsoleteCatalogs = await findCatalogs(context, SYSTEM_USER, syncedCatalogs); + const obsoleteCatalogs = await findAllCatalogsExcluding(context, SYSTEM_USER, syncedCatalogs); if (obsoleteCatalogs.length) { // The app config changed resulting in a catalog not being synced anymore logApp.warn('[OPENCTI-MODULE] Deleting obsolete catalogs', { diff --git a/opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-repository-test.ts b/opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-repository-test.ts index c34a1db23053..f5c11a478dea 100644 --- a/opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-repository-test.ts +++ b/opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-repository-test.ts @@ -1,7 +1,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import { elDeleteInstances, elIndex, elIndexElements } from '../../../../src/database/engine'; -import { internalFindByIdsMapped } from '../../../../src/database/middleware-loader'; -import { deleteCatalogContracts, updateCatalogContracts, upsertCatalog } from '../../../../src/modules/catalog/catalog-repository'; +import { fullEntitiesList, internalFindByIdsMapped } from '../../../../src/database/middleware-loader'; +import { deleteCatalogContracts, findAllCatalogs, findAllCatalogsExcluding, updateCatalogContracts, upsertCatalog } from '../../../../src/modules/catalog/catalog-repository'; import { type BasicStoreEntityCatalog, type BasicStoreEntityCatalogContract, @@ -11,6 +11,7 @@ import { ENTITY_TYPE_CATALOG_CONTRACT, } from '../../../../src/modules/catalog/catalog-types'; import { INDEX_INTERNAL_OBJECTS, READ_INDEX_INTERNAL_OBJECTS } from '../../../../src/database/utils'; +import { FilterMode, FilterOperator } from '../../../../src/generated/graphql'; import type { AuthContext, AuthUser } from '../../../../src/types/user'; vi.mock('../../../../src/database/engine', () => ({ @@ -45,6 +46,53 @@ describe('catalog repository', () => { }); }); + it('should find all catalogs without filters', async () => { + vi.mocked(fullEntitiesList).mockResolvedValue([]); + + await findAllCatalogs(context, user); + + expect(fullEntitiesList).toHaveBeenCalledWith( + context, + user, + [ENTITY_TYPE_CATALOG], + { + indices: [READ_INDEX_INTERNAL_OBJECTS], + filters: null, + }, + ); + }); + + it('should find catalogs excluding the specified catalog ids', async () => { + vi.mocked(fullEntitiesList).mockResolvedValue([]); + + await findAllCatalogsExcluding(context, user, ['catalog-1', 'catalog-2']); + + expect(fullEntitiesList).toHaveBeenCalledWith( + context, + user, + [ENTITY_TYPE_CATALOG], + { + indices: [READ_INDEX_INTERNAL_OBJECTS], + filters: { + filters: [ + { + key: ['catalog_id'], + values: ['catalog-1'], + operator: FilterOperator.NotEq, + }, + { + key: ['catalog_id'], + values: ['catalog-2'], + operator: FilterOperator.NotEq, + }, + ], + filterGroups: [], + mode: FilterMode.And, + }, + }, + ); + }); + it('should upsert a new catalog on the write alias when no existing catalog is provided', async () => { const catalogUpsert = { internal_id: 'catalog--1', diff --git a/opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-sync-domain-test.ts b/opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-sync-domain-test.ts index 3b71cb3b8249..662df066d66e 100644 --- a/opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-sync-domain-test.ts +++ b/opencti-platform/opencti-graphql/tests/01-unit/modules/catalog/catalog-sync-domain-test.ts @@ -7,7 +7,7 @@ const { mockFindCatalogBySourceUri, mockFindCatalogByCatalogId, mockFindCatalogContractsByCatalogId, - mockFindCatalogs, + mockFindAllCatalogsExcluding, mockInsertCatalogContracts, mockUpdateCatalogContracts, mockDeleteCatalogContracts, @@ -22,7 +22,7 @@ const { mockFindCatalogBySourceUri: vi.fn(), mockFindCatalogByCatalogId: vi.fn(), mockFindCatalogContractsByCatalogId: vi.fn(), - mockFindCatalogs: vi.fn(), + mockFindAllCatalogsExcluding: vi.fn(), mockInsertCatalogContracts: vi.fn(), mockUpdateCatalogContracts: vi.fn(), mockDeleteCatalogContracts: vi.fn(), @@ -63,7 +63,7 @@ vi.mock('../../../../src/modules/catalog/catalog-repository', () => ({ findCatalogBySourceUri: mockFindCatalogBySourceUri, findCatalogByCatalogId: mockFindCatalogByCatalogId, findCatalogContractsByCatalogId: mockFindCatalogContractsByCatalogId, - findCatalogs: mockFindCatalogs, + findAllCatalogsExcluding: mockFindAllCatalogsExcluding, insertCatalogContracts: mockInsertCatalogContracts, updateCatalogContracts: mockUpdateCatalogContracts, deleteCatalogContracts: mockDeleteCatalogContracts, @@ -119,7 +119,7 @@ describe('catalog-sync-domain', () => { mockFindCatalogBySourceUri.mockResolvedValue(undefined); mockFindCatalogByCatalogId.mockResolvedValue(undefined); mockFindCatalogContractsByCatalogId.mockResolvedValue(new Map()); - mockFindCatalogs.mockResolvedValue([]); + mockFindAllCatalogsExcluding.mockResolvedValue([]); mockListCatalogContractLogos.mockResolvedValue(new Set()); mockInsertCatalogContracts.mockResolvedValue(undefined); mockUpdateCatalogContracts.mockResolvedValue(undefined); @@ -133,7 +133,7 @@ describe('catalog-sync-domain', () => { const result = await synchronizeCatalogs({ source: 'test' } as any, { id: 'user-1' } as any); expect(result).toEqual(['embedded-catalog']); expect(mockFetchSourceCatalog).toHaveBeenCalledWith({ kind: 'embedded', uri: 'embedded' }, undefined); - expect(mockFindCatalogs).toHaveBeenCalled(); + expect(mockFindAllCatalogsExcluding).toHaveBeenCalled(); }); it('should include custom sources and deduplicate by uri', async () => { @@ -226,7 +226,7 @@ describe('catalog-sync-domain', () => { }); const result = await synchronizeCatalogs({ source: 'test' } as any, { id: 'user-1' } as any); expect(result).toEqual(['embedded-catalog']); - expect(mockFindCatalogs).not.toHaveBeenCalled(); + expect(mockFindAllCatalogsExcluding).not.toHaveBeenCalled(); expect(mockDeleteCatalogs).not.toHaveBeenCalled(); }); }); From fe578d5c6d9658f99893fb745f2516e5824e966a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Thu, 17 Sep 2026 10:49:54 +0200 Subject: [PATCH 35/45] refactor: pass read index to findCatalog repo functions --- .../src/modules/catalog/catalog-repository.ts | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts index 788590a61889..37ae9d8c5fc8 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts @@ -27,7 +27,14 @@ export const findCatalogByCatalogId = async ( user: AuthUser, catalogId: string, ) => { - const catalog = await elLoadBy(context, user, 'catalog_id', catalogId, ENTITY_TYPE_CATALOG); + const catalog = await elLoadBy( + context, + user, + 'catalog_id', + catalogId, + ENTITY_TYPE_CATALOG, + [READ_INDEX_INTERNAL_OBJECTS], + ); return catalog; }; @@ -36,7 +43,14 @@ export const findCatalogBySourceUri = async ( user: AuthUser, sourceUri: string, ) => { - const catalog = await elLoadBy(context, user, 'source_uri', sourceUri); + const catalog = await elLoadBy( + context, + user, + 'source_uri', + sourceUri, + ENTITY_TYPE_CATALOG, + [READ_INDEX_INTERNAL_OBJECTS], + ); return catalog; }; From a5cca8d4498779c06c00d3fa7ff2d53b8768e028 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Thu, 17 Sep 2026 10:50:36 +0200 Subject: [PATCH 36/45] fixup! refactor(catalog): clarify catalog lookup functions (#16059) --- .../opencti-graphql/src/modules/catalog/catalog-repository.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts index 37ae9d8c5fc8..6729abbcd016 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts @@ -1,4 +1,3 @@ -import type { FilterGroup } from '@mistralai/mistralai/models/components'; import { logApp, PLATFORM_VERSION } from '../../config/conf'; import { FunctionalError } from '../../config/errors'; import { elDeleteInstances, elIndex, elIndexElements, elLoadBy } from '../../database/engine'; From 27ec3661f242ca42cc0e0ad9767a12e3de3a468d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Thu, 17 Sep 2026 11:18:36 +0200 Subject: [PATCH 37/45] fix: undo unintended default.json changes --- .../opencti-graphql/config/default.json | 27 ++++--------------- 1 file changed, 5 insertions(+), 22 deletions(-) diff --git a/opencti-platform/opencti-graphql/config/default.json b/opencti-platform/opencti-graphql/config/default.json index 126878c4db9d..a7a9463c6952 100644 --- a/opencti-platform/opencti-graphql/config/default.json +++ b/opencti-platform/opencti-graphql/config/default.json @@ -43,13 +43,8 @@ "logs_console": true, "logs_max_files": 7, "logs_directory": "./logs", - "logs_in_transports": [ - "administration" - ], - "trace_request_headers": [ - "user-agent", - "x-forwarded-for" - ] + "logs_in_transports": ["administration"], + "trace_request_headers": ["user-agent", "x-forwarded-for"] }, "event_loop_logs": { "enabled": false, @@ -185,19 +180,11 @@ }, "groups": { "enabled": true, - "protected_names": [ - "Administrators", - "Connectors", - "Default" - ] + "protected_names": ["Administrators", "Connectors", "Default"] }, "roles": { "enabled": true, - "protected_names": [ - "Administrator", - "Connector", - "Default" - ] + "protected_names": ["Administrator", "Connector", "Default"] }, "rules": { "enabled": true @@ -411,8 +398,6 @@ "engine_selector": "auto", "engine_check": true, "index_creation_pattern": "-000001", - "search_wildcard_prefix": false, - "search_fuzzy": false, "max_pagination_result": 5000, "default_pagination_result": 500, "max_bulk_operations": 5000, @@ -428,9 +413,7 @@ "access_key": "ChangeMe", "secret_key": "ChangeMe", "use_aws_role": false, - "excluded_files": [ - ".DS_Store" - ], + "excluded_files": [".DS_Store"], "disable_checksum_validation": false }, "rabbitmq": { From deb317a36f05b095e75c2bed340a9ec8bdaf21e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Thu, 17 Sep 2026 11:22:21 +0200 Subject: [PATCH 38/45] feat: add info log when processing connector migration --- .../1786264797376-embed-catalog-contract-in-connector-entity.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/opencti-platform/opencti-graphql/src/migrations/1786264797376-embed-catalog-contract-in-connector-entity.ts b/opencti-platform/opencti-graphql/src/migrations/1786264797376-embed-catalog-contract-in-connector-entity.ts index 02c45f915d3f..40895a08d3e4 100644 --- a/opencti-platform/opencti-graphql/src/migrations/1786264797376-embed-catalog-contract-in-connector-entity.ts +++ b/opencti-platform/opencti-graphql/src/migrations/1786264797376-embed-catalog-contract-in-connector-entity.ts @@ -38,6 +38,7 @@ export const up = async (next: (error?: Error) => void) => { const storedLogos = managedConnectors.length > 0 ? (await listCatalogContractLogos()) : new Set(); for (let i = 0; i < managedConnectors.length; i += 1) { const connector = managedConnectors[i]; + logMigration.info(`${message} > processing connector ${connector.id} (${i + 1}/${managedConnectors.length})`); const patch: Pick = {}; if (!connector.manager_contract && connector.manager_contract_image) { const contractDto = contractsByImage[connector.manager_contract_image]; From 57635a29d35deeaf205b25ac7f76806f42c7316c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Thu, 17 Sep 2026 11:31:36 +0200 Subject: [PATCH 39/45] fix: add processing time in migration log --- ...1786264797376-embed-catalog-contract-in-connector-entity.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/opencti-platform/opencti-graphql/src/migrations/1786264797376-embed-catalog-contract-in-connector-entity.ts b/opencti-platform/opencti-graphql/src/migrations/1786264797376-embed-catalog-contract-in-connector-entity.ts index 40895a08d3e4..af61cb5356b7 100644 --- a/opencti-platform/opencti-graphql/src/migrations/1786264797376-embed-catalog-contract-in-connector-entity.ts +++ b/opencti-platform/opencti-graphql/src/migrations/1786264797376-embed-catalog-contract-in-connector-entity.ts @@ -16,6 +16,7 @@ import { mapCatalogContractDtoToCatalogContractSyncSource } from '../modules/cat const message = '[MIGRATION] managed connectors contract snapshot'; export const up = async (next: (error?: Error) => void) => { + const start = new Date().getTime(); logMigration.info(`${message} > started`); const context = executionContext('migration'); @@ -74,7 +75,7 @@ export const up = async (next: (error?: Error) => void) => { logMigration.info(`${message} > connector ${connector.id} patched`); } } - logMigration.info(`${message} > done`); + logMigration.info(`${message} > done in ${new Date().getTime() - start} ms`); next(); }; From 71e7e8cd49c5b8ea771f30a0b2ab3ad5af3aef44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Thu, 17 Sep 2026 11:35:18 +0200 Subject: [PATCH 40/45] fix: add warn log when no compatible contract found during auto-upgrade --- .../src/modules/connector/connector-domain.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/opencti-platform/opencti-graphql/src/modules/connector/connector-domain.ts b/opencti-platform/opencti-graphql/src/modules/connector/connector-domain.ts index b9d24fd93bcf..20901a90837a 100644 --- a/opencti-platform/opencti-graphql/src/modules/connector/connector-domain.ts +++ b/opencti-platform/opencti-graphql/src/modules/connector/connector-domain.ts @@ -32,8 +32,10 @@ const autoUpgradeManagedConnector = async ( if (!latestCompatibleContract) { // Warning: we're running a connector that's not compatible anymore but // there's no replacement version compatible ! - // TODO: stop the connector and block the restart from the UI, or delete - // the connector ? + logApp.warn('[OPENCTI-MODULE] Unable to find a compatible contract when applying auto-upgrade-to-latest-compatible strategy', { + module: 'connector', + connectorId: managedConnector.id, + }); return; } if (semver.eq(contract_version, latestCompatibleContract.contract_version) From 0bdd55ffd269e7544036ea8517d11c5fabd99994 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Thu, 17 Sep 2026 11:38:44 +0200 Subject: [PATCH 41/45] fix: rename migration file --- ...> 1789637887081-embed-catalog-contract-in-connector-entity.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename opencti-platform/opencti-graphql/src/migrations/{1786264797376-embed-catalog-contract-in-connector-entity.ts => 1789637887081-embed-catalog-contract-in-connector-entity.ts} (100%) diff --git a/opencti-platform/opencti-graphql/src/migrations/1786264797376-embed-catalog-contract-in-connector-entity.ts b/opencti-platform/opencti-graphql/src/migrations/1789637887081-embed-catalog-contract-in-connector-entity.ts similarity index 100% rename from opencti-platform/opencti-graphql/src/migrations/1786264797376-embed-catalog-contract-in-connector-entity.ts rename to opencti-platform/opencti-graphql/src/migrations/1789637887081-embed-catalog-contract-in-connector-entity.ts From 1e3557975e0cb27f2a4524915dc5f43f97997852 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Thu, 17 Sep 2026 11:47:41 +0200 Subject: [PATCH 42/45] fix(engine): use isObjectAttributeWithMappings helper to clarify intention --- .../opencti-graphql/src/schema/schema-attributes.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/opencti-platform/opencti-graphql/src/schema/schema-attributes.ts b/opencti-platform/opencti-graphql/src/schema/schema-attributes.ts index cfd5625d17a2..d276ada77957 100644 --- a/opencti-platform/opencti-graphql/src/schema/schema-attributes.ts +++ b/opencti-platform/opencti-graphql/src/schema/schema-attributes.ts @@ -2,7 +2,7 @@ import * as R from 'ramda'; import { GraphQLDateTime } from 'graphql-scalars'; import { RULE_PREFIX } from './general'; import { FunctionalError, UnsupportedError } from '../config/errors'; -import type { AttributeDefinition, AttrType, ComplexAttributeWithMappings, MappingDefinition } from './attribute-definition'; +import type { AttributeDefinition, AttrType, ComplexAttribute, ComplexAttributeWithMappings, MappingDefinition } from './attribute-definition'; import { shortStringFormats } from './attribute-definition'; import { getParentTypes } from './schemaUtils'; import { isFeatureEnabled } from '../config/conf'; @@ -145,7 +145,7 @@ export const schemaAttributesDefinition = { }); } // Check sortBy on object - if (attribute.type === 'object' && attribute.format !== 'flat' && attribute.format !== 'raw' && attribute.sortBy) { + if (isObjectAttributeWithMappings(attribute) && attribute.sortBy) { const correspondingMapping = getAttributeMappingFromPath(attribute.sortBy.path, attribute); if (correspondingMapping.type !== attribute.sortBy.type) { throw UnsupportedError('You can\'t define a sortBy with path and type that do not match the corresponding mapping', { @@ -155,7 +155,7 @@ export const schemaAttributesDefinition = { } } let registeredAttribute: AttributeDefinition = { ...attribute }; - if (registeredAttribute.type === 'object' && registeredAttribute.format !== 'flat' && registeredAttribute.format !== 'raw' && registeredAttribute.mappings.length > 0) { + if (isObjectAttributeWithMappings(registeredAttribute) && registeredAttribute.mappings.length > 0) { registeredAttribute = { ...registeredAttribute, // filter feature flagged attributes From b8031fb3cf7e27e4c7d22aaa74f05ee29281b5f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Thu, 17 Sep 2026 12:00:25 +0200 Subject: [PATCH 43/45] fix: add entity_type check when deleting catalogs --- .../src/modules/catalog/catalog-repository.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts index 6729abbcd016..8db64eedda3f 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts @@ -1,5 +1,5 @@ import { logApp, PLATFORM_VERSION } from '../../config/conf'; -import { FunctionalError } from '../../config/errors'; +import { FunctionalError, UnknownError } from '../../config/errors'; import { elDeleteInstances, elIndex, elIndexElements, elLoadBy } from '../../database/engine'; import { fullEntitiesList, internalFindByIdsMapped, type FilterGroupWithNested } from '../../database/middleware-loader'; import { INDEX_INTERNAL_OBJECTS, READ_INDEX_INTERNAL_OBJECTS } from '../../database/utils'; @@ -99,6 +99,14 @@ export const findAllCatalogsExcluding = async ( }; export const deleteCatalogs = async (context: AuthContext, catalogEntities: BasicStoreEntityCatalog[]) => { + for (let idx = 0; idx < catalogEntities.length; ++idx) { + const catalogEntity = catalogEntities[idx]; + if (catalogEntity.entity_type !== ENTITY_TYPE_CATALOG) { + throw UnknownError(`Expected the entity for deletion to be of type ${ENTITY_TYPE_CATALOG}`, { + actualType: catalogEntity.entity_type, + }); + } + } await elDeleteInstances(context, catalogEntities); }; From 8b9e06449afe01d8e0b18d7b2f450ed567f43e8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Wauquier?= Date: Thu, 17 Sep 2026 12:21:33 +0200 Subject: [PATCH 44/45] feat: add CATALOG_MANAGER translation keys --- opencti-platform/opencti-front/lang/front/de.json | 1 + opencti-platform/opencti-front/lang/front/en.json | 1 + opencti-platform/opencti-front/lang/front/es.json | 1 + opencti-platform/opencti-front/lang/front/fr.json | 1 + opencti-platform/opencti-front/lang/front/it.json | 1 + opencti-platform/opencti-front/lang/front/ja.json | 1 + opencti-platform/opencti-front/lang/front/ko.json | 1 + opencti-platform/opencti-front/lang/front/ru.json | 1 + opencti-platform/opencti-front/lang/front/zh.json | 1 + 9 files changed, 9 insertions(+) diff --git a/opencti-platform/opencti-front/lang/front/de.json b/opencti-platform/opencti-front/lang/front/de.json index 6456ef123f42..28a38a584d80 100644 --- a/opencti-platform/opencti-front/lang/front/de.json +++ b/opencti-platform/opencti-front/lang/front/de.json @@ -639,6 +639,7 @@ "Cases": "Fälle", "Cases & Analyses": "Fälle & Analysen", "CaseTemplate": "CaseTemplate", + "CATALOG_MANAGER": "Katalogmanager", "Category": "Kategorie", "category": "Kategorie", "Center latitude": "Mittlere geographische Breite", diff --git a/opencti-platform/opencti-front/lang/front/en.json b/opencti-platform/opencti-front/lang/front/en.json index 82ee7bfaa1b9..8615ef712106 100644 --- a/opencti-platform/opencti-front/lang/front/en.json +++ b/opencti-platform/opencti-front/lang/front/en.json @@ -639,6 +639,7 @@ "Cases": "Cases", "Cases & Analyses": "Cases & Analyses", "CaseTemplate": "CaseTemplate", + "CATALOG_MANAGER": "Catalog manager", "Category": "Category", "category": "Category", "Center latitude": "Center latitude", diff --git a/opencti-platform/opencti-front/lang/front/es.json b/opencti-platform/opencti-front/lang/front/es.json index fdc3db47522a..30ac9dabf0a4 100644 --- a/opencti-platform/opencti-front/lang/front/es.json +++ b/opencti-platform/opencti-front/lang/front/es.json @@ -639,6 +639,7 @@ "Cases": "Casos", "Cases & Analyses": "Casos y Análisis", "CaseTemplate": "CaseTemplate", + "CATALOG_MANAGER": "Gestor del catálogo", "Category": "Catégorie", "category": "Categoría", "Center latitude": "Latitud central", diff --git a/opencti-platform/opencti-front/lang/front/fr.json b/opencti-platform/opencti-front/lang/front/fr.json index 9922f5673617..cb48ad9eeddc 100644 --- a/opencti-platform/opencti-front/lang/front/fr.json +++ b/opencti-platform/opencti-front/lang/front/fr.json @@ -639,6 +639,7 @@ "Cases": "Cases", "Cases & Analyses": "Cases et analyse", "CaseTemplate": "Modèle de cas", + "CATALOG_MANAGER": "Manager du catalogue", "Category": "Catégorie", "category": "Catégorie", "Center latitude": "Latitude centrale", diff --git a/opencti-platform/opencti-front/lang/front/it.json b/opencti-platform/opencti-front/lang/front/it.json index fbb4c8409053..0afd9f5b8e86 100644 --- a/opencti-platform/opencti-front/lang/front/it.json +++ b/opencti-platform/opencti-front/lang/front/it.json @@ -639,6 +639,7 @@ "Cases": "Casi", "Cases & Analyses": "Casi & Analisi", "CaseTemplate": "Modello di caso", + "CATALOG_MANAGER": "Gestione del catalogo", "Category": "Categoria", "category": "categoria", "Center latitude": "Latitudine centrale", diff --git a/opencti-platform/opencti-front/lang/front/ja.json b/opencti-platform/opencti-front/lang/front/ja.json index 3c24758d64b8..3271d829325c 100644 --- a/opencti-platform/opencti-front/lang/front/ja.json +++ b/opencti-platform/opencti-front/lang/front/ja.json @@ -639,6 +639,7 @@ "Cases": "事例", "Cases & Analyses": "事例と分析", "CaseTemplate": "ケーステンプレート", + "CATALOG_MANAGER": "カタログマネージャー", "Category": "カテゴリー", "category": "カテゴリー", "Center latitude": "中心緯度", diff --git a/opencti-platform/opencti-front/lang/front/ko.json b/opencti-platform/opencti-front/lang/front/ko.json index 56186620ae2d..869a0396e50d 100644 --- a/opencti-platform/opencti-front/lang/front/ko.json +++ b/opencti-platform/opencti-front/lang/front/ko.json @@ -639,6 +639,7 @@ "Cases": "사례", "Cases & Analyses": "사례 및 분석", "CaseTemplate": "케이스 템플릿", + "CATALOG_MANAGER": "카탈로그 관리자", "Category": "카테고리", "category": "카테고리", "Center latitude": "중심 위도", diff --git a/opencti-platform/opencti-front/lang/front/ru.json b/opencti-platform/opencti-front/lang/front/ru.json index 79b98d6c202c..a4664f1f96a1 100644 --- a/opencti-platform/opencti-front/lang/front/ru.json +++ b/opencti-platform/opencti-front/lang/front/ru.json @@ -639,6 +639,7 @@ "Cases": "Кейсы", "Cases & Analyses": "Кейсы и анализы", "CaseTemplate": "CaseTemplate", + "CATALOG_MANAGER": "Менеджер каталога", "Category": "Категория", "category": "категория", "Center latitude": "Центральная широта", diff --git a/opencti-platform/opencti-front/lang/front/zh.json b/opencti-platform/opencti-front/lang/front/zh.json index f92dff994ee7..1205a233f77d 100644 --- a/opencti-platform/opencti-front/lang/front/zh.json +++ b/opencti-platform/opencti-front/lang/front/zh.json @@ -639,6 +639,7 @@ "Cases": "案例", "Cases & Analyses": "案例与分析", "CaseTemplate": "案例模板", + "CATALOG_MANAGER": "目录管理器", "Category": "类别", "category": "类别", "Center latitude": "中心纬度", From ca82fa20aadede51480e777f7af585ddc985c8eb Mon Sep 17 00:00:00 2001 From: Esrevi Huynh Date: Thu, 17 Sep 2026 14:55:51 +0200 Subject: [PATCH 45/45] feat(catalog): reload catalog when manifest has changed on backend (#18280) --- .../components/integrations/Integrations.tsx | 38 +++- .../available/IntegrationsAvailable.tsx | 16 +- .../catalog/IngestionConnectorsCatalog.tsx | 9 + .../integrations/catalog/catalog-constants.ts | 1 + .../catalog/hooks/useCatalogPolling.test.ts | 181 ++++++++++++++++++ .../catalog/hooks/useCatalogPolling.ts | 121 ++++++++++++ .../opencti-front/src/relay/environment.tsx | 5 +- .../src/schema/relay.schema.graphql | 10 +- .../opencti-graphql/src/generated/graphql.ts | 16 ++ .../src/modules/catalog/catalog-domain.ts | 27 ++- .../src/modules/catalog/catalog-repository.ts | 31 +++ .../src/modules/catalog/catalog-resolver.ts | 6 +- .../src/modules/catalog/catalog-types.ts | 5 + .../src/modules/catalog/catalog.graphql | 7 + .../catalog/catalog-resolver-test.ts | 27 +++ 15 files changed, 486 insertions(+), 14 deletions(-) create mode 100644 opencti-platform/opencti-front/src/private/components/integrations/catalog/catalog-constants.ts create mode 100644 opencti-platform/opencti-front/src/private/components/integrations/catalog/hooks/useCatalogPolling.test.ts create mode 100644 opencti-platform/opencti-front/src/private/components/integrations/catalog/hooks/useCatalogPolling.ts diff --git a/opencti-platform/opencti-front/src/private/components/integrations/Integrations.tsx b/opencti-platform/opencti-front/src/private/components/integrations/Integrations.tsx index b67a44642ea2..2d484011e5df 100644 --- a/opencti-platform/opencti-front/src/private/components/integrations/Integrations.tsx +++ b/opencti-platform/opencti-front/src/private/components/integrations/Integrations.tsx @@ -1,5 +1,5 @@ -import React, { Suspense, useEffect, useMemo } from 'react'; -import { Link, Navigate, useParams } from 'react-router'; +import React, { Suspense, useCallback, useEffect, useMemo } from 'react'; +import { Link, Navigate, useParams } from 'react-router'; import { Box, Stack, Typography } from '@mui/material'; import { alpha, useTheme } from '@mui/material/styles'; import { useQueryLoader } from 'react-relay'; @@ -42,6 +42,7 @@ export interface IntegrationsData { deploymentData: IngestionConnectorsQuery['response'] | null; feedsData: IngestionFeedsData | null; formsData: IngestionFeedsFormsData | null; + refetchCatalogs: () => void; refetchFeeds: () => void; refetchForms: () => void; } @@ -78,16 +79,21 @@ const IntegrationsDataProvider = ({ children }: IntegrationsDataProviderProps) = // store-and-network: the previous data keeps rendering while the refresh // happens in the background, so refetching never suspends the whole page. - const refetchFeeds = () => { + const refetchFeeds = useCallback(() => { if (isIngestionReader) { loadFeeds({ first: FEEDS_PAGE_SIZE }, { fetchPolicy: 'store-and-network' }); } - }; - const refetchForms = () => { + }, [isIngestionReader, loadFeeds]); + const refetchForms = useCallback(() => { if (isFormReader) { loadForms({ first: FEEDS_PAGE_SIZE }, { fetchPolicy: 'store-and-network' }); } - }; + }, [isFormReader, loadForms]); + const refetchCatalogs = useCallback(() => { + if (isConnectorReader) { + loadCatalogs({}, { fetchPolicy: 'store-and-network' }); + } + }, [isConnectorReader, loadCatalogs]); const renderWithForms = ( catalogsData: IngestionConnectorsCatalogsQuery['response'] | null, @@ -97,11 +103,27 @@ const IntegrationsDataProvider = ({ children }: IntegrationsDataProviderProps) = if (formsRef) { return ( - {({ data: formsData }) => children({ catalogsData, deploymentData, feedsData, formsData, refetchFeeds, refetchForms })} + {({ data: formsData }) => children({ + catalogsData, + deploymentData, + feedsData, + formsData, + refetchCatalogs, + refetchFeeds, + refetchForms, + })} ); } - return children({ catalogsData, deploymentData, feedsData, formsData: null, refetchFeeds, refetchForms }); + return children({ + catalogsData, + deploymentData, + feedsData, + formsData: null, + refetchCatalogs, + refetchFeeds, + refetchForms, + }); }; const renderWithFeeds = ( diff --git a/opencti-platform/opencti-front/src/private/components/integrations/available/IntegrationsAvailable.tsx b/opencti-platform/opencti-front/src/private/components/integrations/available/IntegrationsAvailable.tsx index 5cd197461fcd..1397f8fb2935 100644 --- a/opencti-platform/opencti-front/src/private/components/integrations/available/IntegrationsAvailable.tsx +++ b/opencti-platform/opencti-front/src/private/components/integrations/available/IntegrationsAvailable.tsx @@ -10,6 +10,7 @@ import IngestionCatalogCard from '@components/integrations/catalog/IngestionCata import IngestionCatalogConnectorCreation from '@components/integrations/catalog/IngestionCatalogConnectorCreation'; import IngestionCatalogFacetSidebar from '@components/integrations/catalog/IngestionCatalogFacetSidebar'; import useConnectorDeployDialog from '@components/integrations/catalog/hooks/useConnectorDeployDialog'; +import useCatalogPolling from '@components/integrations/catalog/hooks/useCatalogPolling'; import useIngestionCatalogFilters, { BUILT_IN_SECTION_KEY, BuiltInCatalogInput, @@ -46,7 +47,20 @@ const IntegrationsAvailable = ({ data }: IntegrationsAvailableProps) => { const theme = useTheme(); const [searchParams] = useSearchParams(); const { hasActiveManagers } = useConnectorManagerStatus(); - const { catalogsData, deploymentData, feedsData, formsData, refetchFeeds, refetchForms } = data; + const { + catalogsData, + deploymentData, + feedsData, + formsData, + refetchCatalogs, + refetchFeeds, + refetchForms, + } = data; + + useCatalogPolling({ + enabled: catalogsData !== null, + onCatalogRevisionsChanged: refetchCatalogs, + }); const { catalogState, handleOpenDeployDialog, handleCloseDeployDialog, handleCreate } = useConnectorDeployDialog(); const [builtInCreationKind, setBuiltInCreationKind] = useState(null); diff --git a/opencti-platform/opencti-front/src/private/components/integrations/catalog/IngestionConnectorsCatalog.tsx b/opencti-platform/opencti-front/src/private/components/integrations/catalog/IngestionConnectorsCatalog.tsx index 82261c51e4f3..7fc8f1a48b7a 100644 --- a/opencti-platform/opencti-front/src/private/components/integrations/catalog/IngestionConnectorsCatalog.tsx +++ b/opencti-platform/opencti-front/src/private/components/integrations/catalog/IngestionConnectorsCatalog.tsx @@ -3,6 +3,15 @@ import type { PreloadedQuery } from 'react-relay'; import React from 'react'; import { IngestionConnectorsCatalogsQuery } from '@components/integrations/catalog/__generated__/IngestionConnectorsCatalogsQuery.graphql'; +export const ingestionConnectorsCatalogRevisionsQuery = graphql` + query IngestionConnectorsCatalogRevisionsQuery { + catalogsRevisions { + catalog_id + revision + } + } +`; + export const ingestionConnectorsCatalogsQuery = graphql` query IngestionConnectorsCatalogsQuery { catalogs { diff --git a/opencti-platform/opencti-front/src/private/components/integrations/catalog/catalog-constants.ts b/opencti-platform/opencti-front/src/private/components/integrations/catalog/catalog-constants.ts new file mode 100644 index 000000000000..9b441047a314 --- /dev/null +++ b/opencti-platform/opencti-front/src/private/components/integrations/catalog/catalog-constants.ts @@ -0,0 +1 @@ +export const CATALOG_POLLING_INTERVAL_MS = 60000; diff --git a/opencti-platform/opencti-front/src/private/components/integrations/catalog/hooks/useCatalogPolling.test.ts b/opencti-platform/opencti-front/src/private/components/integrations/catalog/hooks/useCatalogPolling.test.ts new file mode 100644 index 000000000000..07dd1a4b4d9c --- /dev/null +++ b/opencti-platform/opencti-front/src/private/components/integrations/catalog/hooks/useCatalogPolling.test.ts @@ -0,0 +1,181 @@ +import { act, renderHook } from '@testing-library/react'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { CATALOG_POLLING_INTERVAL_MS } from '../catalog-constants'; +import useCatalogPolling from './useCatalogPolling'; + +const mocks = vi.hoisted(() => ({ + fetchQuery: vi.fn(), +})); + +vi.mock('../../../../../relay/environment', () => ({ + fetchQuery: mocks.fetchQuery, +})); + +type RevisionsPayload = { + catalogsRevisions: Array<{ catalog_id: string; revision: string | null }>; +}; + +const setDocumentHidden = (hidden: boolean) => { + Object.defineProperty(document, 'hidden', { + configurable: true, + get: () => hidden, + }); +}; + +const mockFetchSequence = (steps: Array) => { + let index = 0; + mocks.fetchQuery.mockImplementation(() => ({ + toPromise: () => { + const current = steps[Math.min(index, steps.length - 1)]; + index += 1; + if (current instanceof Error) { + return Promise.reject(current); + } + return Promise.resolve(current); + }, + })); +}; + +const flushPromises = async () => { + await Promise.resolve(); + await Promise.resolve(); +}; + +describe('useCatalogPolling', () => { + beforeEach(() => { + vi.useFakeTimers(); + vi.clearAllMocks(); + setDocumentHidden(false); + }); + + afterEach(() => { + vi.useRealTimers(); + vi.restoreAllMocks(); + }); + + it('seeds baseline without triggering a full catalogs refresh', async () => { + mockFetchSequence([ + { catalogsRevisions: [{ catalog_id: 'catalog-1', revision: 'rev-1' }] }, + { catalogsRevisions: [{ catalog_id: 'catalog-1', revision: 'rev-1' }] }, + ]); + const onChanged = vi.fn(); + + renderHook(() => useCatalogPolling({ enabled: true, onCatalogRevisionsChanged: onChanged })); + await act(async () => { + await flushPromises(); + }); + + expect(mocks.fetchQuery).toHaveBeenCalledWith( + expect.anything(), + {}, + { fetchPolicy: 'network-only' }, + ); + + expect(onChanged).not.toHaveBeenCalled(); + + await act(async () => { + vi.advanceTimersByTime(CATALOG_POLLING_INTERVAL_MS); + await flushPromises(); + }); + + expect(onChanged).not.toHaveBeenCalled(); + }); + + it('refreshes catalogs when a revision changes and updates baseline', async () => { + mockFetchSequence([ + { catalogsRevisions: [{ catalog_id: 'catalog-1', revision: 'rev-1' }] }, + { catalogsRevisions: [{ catalog_id: 'catalog-1', revision: 'rev-2' }] }, + { catalogsRevisions: [{ catalog_id: 'catalog-1', revision: 'rev-2' }] }, + ]); + const onChanged = vi.fn(); + + renderHook(() => useCatalogPolling({ enabled: true, onCatalogRevisionsChanged: onChanged })); + await act(async () => { + await flushPromises(); + }); + + await act(async () => { + vi.advanceTimersByTime(CATALOG_POLLING_INTERVAL_MS); + await flushPromises(); + }); + expect(onChanged).toHaveBeenCalledTimes(1); + + await act(async () => { + vi.advanceTimersByTime(CATALOG_POLLING_INTERVAL_MS); + await flushPromises(); + }); + expect(onChanged).toHaveBeenCalledTimes(1); + }); + + it('retries after a failed seed without triggering a refresh', async () => { + mockFetchSequence([ + new Error('network down'), + { catalogsRevisions: [{ catalog_id: 'catalog-1', revision: 'rev-1' }] }, + { catalogsRevisions: [{ catalog_id: 'catalog-1', revision: 'rev-1' }] }, + ]); + const onChanged = vi.fn(); + + renderHook(() => useCatalogPolling({ enabled: true, onCatalogRevisionsChanged: onChanged })); + await act(async () => { + await flushPromises(); + }); + + await act(async () => { + vi.advanceTimersByTime(CATALOG_POLLING_INTERVAL_MS); + await flushPromises(); + }); + + expect(onChanged).not.toHaveBeenCalled(); + expect(mocks.fetchQuery).toHaveBeenCalledTimes(2); + }); + + it('pauses polling while hidden and performs an immediate check when visible again', async () => { + mockFetchSequence([ + { catalogsRevisions: [{ catalog_id: 'catalog-1', revision: 'rev-1' }] }, + { catalogsRevisions: [{ catalog_id: 'catalog-1', revision: 'rev-1' }] }, + { catalogsRevisions: [{ catalog_id: 'catalog-1', revision: 'rev-1' }] }, + ]); + + renderHook(() => useCatalogPolling({ enabled: true, onCatalogRevisionsChanged: vi.fn() })); + await act(async () => { + await flushPromises(); + }); + expect(mocks.fetchQuery).toHaveBeenCalledTimes(1); + + await act(async () => { + setDocumentHidden(true); + document.dispatchEvent(new Event('visibilitychange')); + vi.advanceTimersByTime(CATALOG_POLLING_INTERVAL_MS * 2); + await flushPromises(); + }); + expect(mocks.fetchQuery).toHaveBeenCalledTimes(1); + + await act(async () => { + setDocumentHidden(false); + document.dispatchEvent(new Event('visibilitychange')); + await flushPromises(); + }); + expect(mocks.fetchQuery).toHaveBeenCalledTimes(2); + }); + + it('stops polling on unmount', async () => { + mockFetchSequence([ + { catalogsRevisions: [{ catalog_id: 'catalog-1', revision: 'rev-1' }] }, + { catalogsRevisions: [{ catalog_id: 'catalog-1', revision: 'rev-1' }] }, + ]); + + const { unmount } = renderHook(() => useCatalogPolling({ enabled: true, onCatalogRevisionsChanged: vi.fn() })); + await act(async () => { + await flushPromises(); + }); + + unmount(); + + await act(async () => { + vi.advanceTimersByTime(CATALOG_POLLING_INTERVAL_MS * 2); + await flushPromises(); + }); + + expect(mocks.fetchQuery).toHaveBeenCalledTimes(1); + }); +}); diff --git a/opencti-platform/opencti-front/src/private/components/integrations/catalog/hooks/useCatalogPolling.ts b/opencti-platform/opencti-front/src/private/components/integrations/catalog/hooks/useCatalogPolling.ts new file mode 100644 index 000000000000..012fd9b78ecc --- /dev/null +++ b/opencti-platform/opencti-front/src/private/components/integrations/catalog/hooks/useCatalogPolling.ts @@ -0,0 +1,121 @@ +import { useEffect, useRef } from 'react'; +import { fetchQuery } from '../../../../../relay/environment'; +import { ingestionConnectorsCatalogRevisionsQuery } from '../IngestionConnectorsCatalog'; +import { CATALOG_POLLING_INTERVAL_MS } from '../catalog-constants'; +import type { IngestionConnectorsCatalogRevisionsQuery } from '../__generated__/IngestionConnectorsCatalogRevisionsQuery.graphql'; + +type UseCatalogPollingProps = { + enabled: boolean; + onCatalogRevisionsChanged: () => Promise | void; +}; + +type RevisionByCatalogId = Map; + +const toRevisionMap = ( + revisions: ReadonlyArray<{ catalog_id: string; revision: string | null | undefined }> | null | undefined, +): RevisionByCatalogId => { + return new Map((revisions ?? []).map((entry) => [entry.catalog_id, entry.revision ?? null])); +}; + +const haveRevisionsChanged = (baseline: RevisionByCatalogId, next: RevisionByCatalogId): boolean => { + if (baseline.size !== next.size) { + return true; + } + for (const [catalogId, revision] of next) { + if (!baseline.has(catalogId) || baseline.get(catalogId) !== revision) { + return true; + } + } + return false; +}; + +const useCatalogPolling = ({ enabled, onCatalogRevisionsChanged }: UseCatalogPollingProps) => { + const baselineRef = useRef(null); + const timeoutRef = useRef | null>(null); + const isUnmountedRef = useRef(false); + const isCheckInFlightRef = useRef(false); + const wasPausedRef = useRef(false); + + useEffect(() => { + if (!enabled) { + return undefined; + } + + isUnmountedRef.current = false; + + const clearScheduledCheck = () => { + if (timeoutRef.current) { + clearTimeout(timeoutRef.current); + timeoutRef.current = null; + } + }; + + const scheduleNextCheck = () => { + if (isUnmountedRef.current || document.hidden) { + return; + } + clearScheduledCheck(); + timeoutRef.current = setTimeout(() => { + void checkCatalogRevisions(); + }, CATALOG_POLLING_INTERVAL_MS); + }; + + const checkCatalogRevisions = async () => { + if (isCheckInFlightRef.current || isUnmountedRef.current || document.hidden) { + return; + } + isCheckInFlightRef.current = true; + try { + const result = await fetchQuery( + ingestionConnectorsCatalogRevisionsQuery, + {}, + { fetchPolicy: 'network-only' }, + ).toPromise().catch(() => null); + + if (!result) { + return; + } + + const nextBaseline = toRevisionMap(result.catalogsRevisions ?? []); + + if (!baselineRef.current) { + baselineRef.current = nextBaseline; + return; + } + + if (!haveRevisionsChanged(baselineRef.current, nextBaseline)) { + return; + } + + await onCatalogRevisionsChanged(); + baselineRef.current = nextBaseline; + } finally { + isCheckInFlightRef.current = false; + scheduleNextCheck(); + } + }; + + const onVisibilityChange = () => { + if (document.hidden) { + wasPausedRef.current = true; + clearScheduledCheck(); + return; + } + if (wasPausedRef.current) { + wasPausedRef.current = false; + void checkCatalogRevisions(); + } + }; + + document.addEventListener('visibilitychange', onVisibilityChange); + void checkCatalogRevisions(); + + return () => { + isUnmountedRef.current = true; + clearScheduledCheck(); + document.removeEventListener('visibilitychange', onVisibilityChange); + }; + }, [enabled, onCatalogRevisionsChanged]); +}; + +export default useCatalogPolling; diff --git a/opencti-platform/opencti-front/src/relay/environment.tsx b/opencti-platform/opencti-front/src/relay/environment.tsx index d49a8a1ddf67..a0c2efdc4bba 100644 --- a/opencti-platform/opencti-front/src/relay/environment.tsx +++ b/opencti-platform/opencti-front/src/relay/environment.tsx @@ -121,6 +121,8 @@ interface QueryRendererProps { fetchPolicy?: FetchPolicy; } +type FetchQueryOptions = NonNullable[3]>; + export const QueryRenderer = ({ variables, query, @@ -227,7 +229,8 @@ export const requestSubscription = (args) => RS(environment, args); export const fetchQuery = ( query: GraphQLTaggedNode, args: T['variables'] = {}, -) => FQ(environment, query, args); + options?: FetchQueryOptions, +) => FQ(environment, query, args, options); export const commitLocalUpdate = (updater: SelectorStoreUpdater) => CLU(environment, updater); diff --git a/opencti-platform/opencti-front/src/schema/relay.schema.graphql b/opencti-platform/opencti-front/src/schema/relay.schema.graphql index 291536faf5ea..65caf4c5d5f6 100644 --- a/opencti-platform/opencti-front/src/schema/relay.schema.graphql +++ b/opencti-platform/opencti-front/src/schema/relay.schema.graphql @@ -5585,9 +5585,9 @@ enum ThreatActorsOrdering { updated_at x_opencti_workflow_id confidence + x_opencti_score sophistication resource_level - x_opencti_score _score threat_actor_types objectMarking @@ -5696,6 +5696,7 @@ type ThreatActorGroup implements BasicObject & StixObject & StixCoreObject & Sti metrics: [Metric] representative: Representative! x_opencti_stix_ids: [StixId] + x_opencti_score: Int is_inferred: Boolean! spec_version: String! created_at: DateTime! @@ -5744,7 +5745,6 @@ type ThreatActorGroup implements BasicObject & StixObject & StixCoreObject & Sti goals: [String] sophistication: String resource_level: String - x_opencti_score: Int primary_motivation: String secondary_motivations: [String] personal_motivations: [String] @@ -9469,6 +9469,7 @@ type Query { channels(first: Int, after: ID, orderBy: ChannelsOrdering, orderMode: OrderingMode, filters: FilterGroup, search: String): ChannelConnection catalog(id: String!): Catalog catalogs: [Catalog!]! + catalogsRevisions: [CatalogRevision!]! contract(slug: String!): ExtendedContract language(id: String!): Language languages(first: Int, after: ID, orderBy: LanguagesOrdering, orderMode: OrderingMode, filters: FilterGroup, search: String): LanguageConnection @@ -10968,6 +10969,11 @@ type ExtendedContract { contract: String! } +type CatalogRevision { + catalog_id: String! + revision: String +} + enum CatalogsOrdering { name _score diff --git a/opencti-platform/opencti-graphql/src/generated/graphql.ts b/opencti-platform/opencti-graphql/src/generated/graphql.ts index 8beaa3bc28ac..e796597d9a55 100644 --- a/opencti-platform/opencti-graphql/src/generated/graphql.ts +++ b/opencti-platform/opencti-graphql/src/generated/graphql.ts @@ -3880,6 +3880,12 @@ export type CatalogEdge = { node: Catalog; }; +export type CatalogRevision = { + __typename?: 'CatalogRevision'; + catalog_id: Scalars['String']['output']; + revision?: Maybe; +}; + export enum CatalogsOrdering { Score = '_score', Name = 'name' @@ -24752,6 +24758,7 @@ export type Query = { cases?: Maybe; catalog?: Maybe; catalogs: Array; + catalogsRevisions: Array; channel?: Maybe; channels?: Maybe; cities?: Maybe; @@ -40617,6 +40624,7 @@ export type ResolversTypes = ResolversObject<{ Catalog: ResolverTypeWrapper; CatalogConnection: ResolverTypeWrapper & { edges: Array }>; CatalogEdge: ResolverTypeWrapper & { node: ResolversTypes['Catalog'] }>; + CatalogRevision: ResolverTypeWrapper; CatalogsOrdering: CatalogsOrdering; CertAuthConfig: ResolverTypeWrapper; CertAuthConfigInput: CertAuthConfigInput; @@ -41788,6 +41796,7 @@ export type ResolversParentTypes = ResolversObject<{ Catalog: GraphqlCatalog; CatalogConnection: Omit & { edges: Array }; CatalogEdge: Omit & { node: ResolversParentTypes['Catalog'] }; + CatalogRevision: CatalogRevision; CertAuthConfig: CertAuthConfig; CertAuthConfigInput: CertAuthConfigInput; ChangePasswordInput: ChangePasswordInput; @@ -43954,6 +43963,11 @@ export type CatalogEdgeResolvers; }>; +export type CatalogRevisionResolvers = ResolversObject<{ + catalog_id?: Resolver; + revision?: Resolver, ParentType, ContextType>; +}>; + export type CertAuthConfigResolvers = ResolversObject<{ button_label_override?: Resolver, ParentType, ContextType>; description?: Resolver, ParentType, ContextType>; @@ -50597,6 +50611,7 @@ export type QueryResolvers, ParentType, ContextType, Partial>; catalog?: Resolver, ParentType, ContextType, RequireFields>; catalogs?: Resolver, ParentType, ContextType>; + catalogsRevisions?: Resolver, ParentType, ContextType>; channel?: Resolver, ParentType, ContextType, RequireFields>; channels?: Resolver, ParentType, ContextType, Partial>; cities?: Resolver, ParentType, ContextType, Partial>; @@ -54873,6 +54888,7 @@ export type Resolvers = ResolversObject<{ Catalog?: CatalogResolvers; CatalogConnection?: CatalogConnectionResolvers; CatalogEdge?: CatalogEdgeResolvers; + CatalogRevision?: CatalogRevisionResolvers; CertAuthConfig?: CertAuthConfigResolvers; Channel?: ChannelResolvers; ChannelConnection?: ChannelConnectionResolvers; diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-domain.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-domain.ts index 616491f1fa91..beeb3e9d6b87 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-domain.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-domain.ts @@ -8,13 +8,20 @@ import { type CatalogContract, type CatalogContractEntityFields, type GraphqlCatalog, + type GraphqlCatalogRevision, type GraphqlCatalogContract, } from './catalog-types'; import { isEmptyField } from '../../database/utils'; import { UnsupportedError } from '../../config/errors'; import type { ConnectorContractConfiguration, ContractConfigInput } from '../../generated/graphql'; import type { ValidateFunction } from 'ajv'; -import { findAllCatalogs, findCatalogByCatalogId, findLatestCompatibleCatalogContractBySlug, findLatestCompatibleCatalogContractsByCatalogId } from './catalog-repository'; +import { + findAllCatalogs, + findCatalogByCatalogId, + findCatalogsRevisions, + findLatestCompatibleCatalogContractBySlug, + findLatestCompatibleCatalogContractsByCatalogId, +} from './catalog-repository'; import { logApp } from '../../config/conf'; const validatorCache = new Map(); @@ -466,6 +473,24 @@ export const queryCatalogs = async (context: AuthContext, user: AuthUser) => { return ret; }; +export const findCatalogRevisions = async (context: AuthContext, user: AuthUser): Promise => { + try { + const catalogsRevisions = await findCatalogsRevisions(context, user); + const revisions = catalogsRevisions.map((catalog) => ({ + catalog_id: catalog.catalog_id, + revision: catalog.revision ?? null, + })); + logApp.debug('[OPENCTI-MODULE] [catalog] Catalog revisions query resolved', { + module: 'catalog', + catalogsCount: revisions.length, + }); + return revisions; + } catch (error) { + logApp.error('[OPENCTI-MODULE] [catalog] Catalog revisions query failed', { module: 'catalog', error }); + throw error; + } +}; + export const queryContractBySlug = async (context: AuthContext, user: AuthUser, contractSlug: string) => { const contract = await findLatestCompatibleCatalogContractBySlug(context, user, contractSlug); if (!contract) { diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts index 8db64eedda3f..a9bb1dff67a8 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-repository.ts @@ -98,6 +98,37 @@ export const findAllCatalogsExcluding = async ( return findAllCatalogs(context, user, filters); }; +export const findCatalogsRevisions = async (context: AuthContext, user: AuthUser, excludedIds?: string[]) => { + const filters: FilterGroupWithNested | undefined = excludedIds?.length + ? { + filters: excludedIds.map((catalogId) => ({ + key: ['catalog_id'], + values: [catalogId], + operator: FilterOperator.NotEq, + })), + filterGroups: [], + mode: FilterMode.And, + } + : undefined; + + const catalogs = await fullEntitiesList( + context, + user, + [ENTITY_TYPE_CATALOG], + { + indices: [READ_INDEX_INTERNAL_OBJECTS], + filters, + baseData: true, + baseFields: ['catalog_id', 'revision'], + }, + ); + + return catalogs.map((catalog) => ({ + catalog_id: catalog.catalog_id, + revision: catalog.revision, + })); +}; + export const deleteCatalogs = async (context: AuthContext, catalogEntities: BasicStoreEntityCatalog[]) => { for (let idx = 0; idx < catalogEntities.length; ++idx) { const catalogEntity = catalogEntities[idx]; diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-resolver.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-resolver.ts index 6b215e578d57..a3e3addc0920 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-resolver.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-resolver.ts @@ -1,5 +1,6 @@ import type { Resolvers } from '../../generated/graphql'; -import { queryCatalogById, queryCatalogs, queryContractBySlug } from './catalog-domain'; +import type { AuthContext } from '../../types/user'; +import { findCatalogRevisions, queryCatalogById, queryCatalogs, queryContractBySlug } from './catalog-domain'; const catalogResolver: Resolvers = { Query: { @@ -9,6 +10,9 @@ const catalogResolver: Resolvers = { catalogs: (_, _args, context) => { return queryCatalogs(context, context.user); }, + catalogsRevisions: (_: unknown, _args: Record, context: AuthContext) => { + return findCatalogRevisions(context, context.user!); + }, contract: (_, { slug }, context) => { return queryContractBySlug(context, context.user, slug); }, diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-types.ts b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-types.ts index 0c0e01fff38d..ca0a19f48caa 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog-types.ts +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog-types.ts @@ -76,6 +76,11 @@ export interface GraphqlCatalog { description: string; contracts: string[]; // JSON.Stringified GraphqlCatalogContract items } + +export interface GraphqlCatalogRevision { + catalog_id: string; + revision: string | null; +} // endregion // region Database types diff --git a/opencti-platform/opencti-graphql/src/modules/catalog/catalog.graphql b/opencti-platform/opencti-graphql/src/modules/catalog/catalog.graphql index 651644bd3ace..71503d051696 100644 --- a/opencti-platform/opencti-graphql/src/modules/catalog/catalog.graphql +++ b/opencti-platform/opencti-graphql/src/modules/catalog/catalog.graphql @@ -16,6 +16,11 @@ type ExtendedContract { contract: String! # JSON.Stringified GraphqlCatalogContract items } +type CatalogRevision { + catalog_id: String! + revision: String +} + # Ordering enum CatalogsOrdering { name @@ -40,6 +45,8 @@ type Query { # Returns only the latest compatible version of each contract for # backwards compatibility. catalogs: [Catalog!]! @auth(for: [INGESTION]) + # Lightweight polling endpoint for catalog refresh detection. + catalogsRevisions: [CatalogRevision!]! @auth(for: [INGESTION]) # Returns only the latest compatible version for backwards compatibility. contract(slug: String!): ExtendedContract @auth(for: [INGESTION]) } diff --git a/opencti-platform/opencti-graphql/tests/03-integration/10-modules/catalog/catalog-resolver-test.ts b/opencti-platform/opencti-graphql/tests/03-integration/10-modules/catalog/catalog-resolver-test.ts index e16523a1e021..70ef28b5b81d 100644 --- a/opencti-platform/opencti-graphql/tests/03-integration/10-modules/catalog/catalog-resolver-test.ts +++ b/opencti-platform/opencti-graphql/tests/03-integration/10-modules/catalog/catalog-resolver-test.ts @@ -21,6 +21,15 @@ const LIST_CATALOGS_QUERY = gql` } `; +const LIST_CATALOG_REVISIONS_QUERY = gql` + query CatalogsRevisions { + catalogsRevisions { + catalog_id + revision + } + } +`; + const GET_CATALOG_BY_ID_QUERY = gql` query CatalogById($id: String!) { catalog(id: $id) { @@ -196,4 +205,22 @@ describe('Catalog resolver integration', () => { it('should expose expected endpoint shapes with a custom V1 catalog source', async () => { await assertCatalogEndpoints(catalogV1Id, catalogV1Slug, v1CatalogUri); }); + + it('should expose lightweight catalogs revisions payload', async () => { + conf.set('catalog_manager:custom_catalog_refresh_endpoint_uri', v1CatalogUri); + await synchronizeCatalogs(testContext, ADMIN_USER); + + const result = await queryAsAdminWithSuccess({ + query: LIST_CATALOG_REVISIONS_QUERY, + variables: {}, + }); + const revisions = result.data?.catalogsRevisions ?? []; + expect(revisions.length).toBeGreaterThan(0); + + for (const revisionEntry of revisions) { + expect(Object.keys(revisionEntry).sort()).toEqual(['catalog_id', 'revision']); + expect(revisionEntry.catalog_id).toEqual(expect.any(String)); + expect(revisionEntry.revision).toEqual(expect.any(String)); + } + }); });