From 4da2f96581859027399cb36e971c0428722045e1 Mon Sep 17 00:00:00 2001 From: friedisplay Date: Sun, 6 Sep 2026 22:09:59 +0300 Subject: [PATCH] feat: add occurrence REST API --- .../dto/search-occurrence-query.dto.ts | 16 +++ .../occurrence/occurrence-response.mapper.ts | 99 +++++++++++++++ .../occurrence-search.controller.ts | 115 ++++++++++++++++++ .../src/db/occurrence/occurrence.module.ts | 3 +- .../src/db/occurrence/occurrence.resolver.ts | 109 ++--------------- .../src/db/occurrence/occurrence.service.ts | 100 +++++++++++---- .../src/db/occurrence/search-query.utils.ts | 82 +++++++++++++ src/API/src/schema.gql | 51 +++++++- 8 files changed, 444 insertions(+), 131 deletions(-) create mode 100644 src/API/src/db/occurrence/dto/search-occurrence-query.dto.ts create mode 100644 src/API/src/db/occurrence/occurrence-response.mapper.ts create mode 100644 src/API/src/db/occurrence/occurrence-search.controller.ts create mode 100644 src/API/src/db/occurrence/search-query.utils.ts diff --git a/src/API/src/db/occurrence/dto/search-occurrence-query.dto.ts b/src/API/src/db/occurrence/dto/search-occurrence-query.dto.ts new file mode 100644 index 000000000..da6d65e05 --- /dev/null +++ b/src/API/src/db/occurrence/dto/search-occurrence-query.dto.ts @@ -0,0 +1,16 @@ +export interface SearchOccurrenceQueryDto { + id?: string; + country?: string; + species?: string; + season?: string; + insecticide?: string; + absence?: string; + abundance?: string; + bionomics?: string; + adult?: string; + larval?: string; + timeline?: string; + take?: string; + skip?: string; + fields?: string; +} diff --git a/src/API/src/db/occurrence/occurrence-response.mapper.ts b/src/API/src/db/occurrence/occurrence-response.mapper.ts new file mode 100644 index 000000000..03e8243c2 --- /dev/null +++ b/src/API/src/db/occurrence/occurrence-response.mapper.ts @@ -0,0 +1,99 @@ +import { Occurrence } from './entities/occurrence.entity'; +import { OccurrenceService } from './occurrence.service'; +import { OccurrenceReturn } from './occurrenceReturn'; + +export function mapOccurrencesToReturnItems( + items: Occurrence[], + occurrenceService: OccurrenceService, + minimalFields = true, +): OccurrenceReturn[] { + const relationObject = occurrenceService.getOccurrenceFields(true); + const excludeColumns = { + parent: [], + relations: { + dataset: [ + 'id', + 'status', + 'UpdatedBy', + 'UpdatedAt', + 'ReviewedBy', + 'ReviewedAt', + 'ApprovedBy', + 'ApprovedAt', + ], + site: ['longitude_4', 'longitude_5'], + }, + }; + + const includeColumn = ( + isParentProperty: boolean, + relationName: string, + columnName: string, + ) => { + let cols: any; + if (isParentProperty) { + cols = excludeColumns['parent']; + } else { + cols = excludeColumns['relations'][relationName]; + } + if (cols === '*') return false; + if (Array.isArray(cols) && cols.includes(columnName)) return false; + return true; + }; + + const selectAllFields = (record: object, destinationObject: object) => { + Object.keys(record).map((dataProperty) => { + if (Object.keys(relationObject).includes(dataProperty)) { + const relationFields = relationObject[dataProperty]; + relationFields.map((relationField) => { + if ( + relationField === 'id' || + !includeColumn(false, dataProperty, relationField) + ) { + // omitted + } else { + const key = `${dataProperty}_${relationField}`; + Object.assign(destinationObject, { + [key]: record?.[dataProperty]?.[relationField] || null, + }); + } + }); + } else { + if (includeColumn(true, null, dataProperty)) { + Object.assign(destinationObject, { + [dataProperty]: record?.[dataProperty], + }); + } + } + }); + return destinationObject; + }; + + return items.map((x) => { + const obj = { + id: x.id, + species: x.recordedSpecies.species, + location: x.site.location, + binary_presence: x.binary_presence, + country: x.site.country, + year_start: x.year_start, + is_adult: !!x.adult_data, + is_larval: x.larval_data === 'True', + season_val: x.season_calc || x.season_given || '', + insecticide: x.insecticide_resistance_data, + control: x.sample?.control?.toString() || '', + abundance_data: x.abundance_data, + bio_data: x.bio_data, + display_name: x.recordedSpecies?.display_name, + category: x.recordedSpecies?.category, + color: x.recordedSpecies?.color, + //has_bionomics: x.bio_data, + }; + + if (!minimalFields) { + const extendedObject = selectAllFields(x, obj); + Object.assign(obj, extendedObject); + } + return obj; + }); +} diff --git a/src/API/src/db/occurrence/occurrence-search.controller.ts b/src/API/src/db/occurrence/occurrence-search.controller.ts new file mode 100644 index 000000000..1627d37a2 --- /dev/null +++ b/src/API/src/db/occurrence/occurrence-search.controller.ts @@ -0,0 +1,115 @@ +import { + BadRequestException, + Controller, + Get, + NotFoundException, + Param, + Query, +} from '@nestjs/common'; +import { OccurrenceService } from './occurrence.service'; +import { SearchOccurrenceQueryDto } from './dto/search-occurrence-query.dto'; +import { mapOccurrencesToReturnItems } from './occurrence-response.mapper'; +import { + parseStringArray, + parseNullableStringArray, + parseNullableBooleanArray, + parseTimeline, + parseFields, + projectFields, +} from './search-query.utils'; + +/** + * GET /search/:entityType + * This controller only validates input, translates REST query params into the + * OccurrenceFilter shape, delegates to the existing service, and formats + * the response via the shared mapper (occurrence-response.mapper.ts) so + * REST and GraphQL responses stay identical in shape. + */ +@Controller('search') +export class OccurrenceSearchController { + constructor(private readonly occurrenceService: OccurrenceService) {} + + @Get(':entityType') + async search( + @Param('entityType') entityType: string, + @Query() query: SearchOccurrenceQueryDto, + ) { + // Manual validation + if (entityType.toLowerCase() !== 'occurrence') { + throw new BadRequestException( + `Unsupported entity type "${entityType}". Only "occurrence" is currently supported.`, + ); + } + + const fields = parseFields(query.fields); + + // Single-record lookup by id — bypasses filters/pagination entirely and + // reuses the existing findOneById, + if (query.id) { + const occurrence = await this.occurrenceService.findOneById(query.id); + if (!occurrence) { + throw new NotFoundException( + `No occurrence found with id "${query.id}".`, + ); + } + const [mapped] = mapOccurrencesToReturnItems( + [occurrence], + this.occurrenceService, + true, + ); + return { + items: [projectFields(mapped, fields)], + total: 1, + hasMore: false, + }; + } + + const take = query.take ? parseInt(query.take, 10) : 100; + const skip = query.skip ? parseInt(query.skip, 10) : 0; + if (isNaN(take) || take < 1 || take > 5000) { + throw new BadRequestException( + 'take must be a number between 1 and 5000.', + ); + } + if (isNaN(skip) || skip < 0) { + throw new BadRequestException('skip must be a number >= 0.'); + } + + const filters: any = { + country: parseStringArray(query.country), + species: parseStringArray(query.species), + season: parseNullableStringArray(query.season), + insecticide: parseNullableStringArray(query.insecticide), + binary_presence: parseNullableStringArray(query.absence), + abundance_data: parseNullableStringArray(query.abundance), + bionomics: parseNullableBooleanArray(query.bionomics)?.filter( + (v) => v !== null, + ), + isAdult: parseNullableBooleanArray(query.adult), + isLarval: parseNullableBooleanArray(query.larval), + ...parseTimeline(query.timeline), + }; + + const bounds = { locationWindowActive: false }; + + const { items, total } = await this.occurrenceService.findOccurrences( + take, + skip, + filters, + bounds as any, + true, + ); + + const mappedItems = mapOccurrencesToReturnItems( + items, + this.occurrenceService, + true, + ); + + return { + items: mappedItems.map((item) => projectFields(item, fields)), + total, + hasMore: total > take + skip, + }; + } +} diff --git a/src/API/src/db/occurrence/occurrence.module.ts b/src/API/src/db/occurrence/occurrence.module.ts index 90d3bffbf..fe413b268 100644 --- a/src/API/src/db/occurrence/occurrence.module.ts +++ b/src/API/src/db/occurrence/occurrence.module.ts @@ -27,6 +27,7 @@ import { AuthService } from 'src/auth/auth.service'; import { UserRoleService } from 'src/auth/user_role/user_role.service'; import { UserRole } from 'src/auth/user_role/user_role.entity'; import { OccurrenceController } from './occurrence.controller'; +import { OccurrenceSearchController } from './occurrence-search.controller'; import { Dataset } from '../shared/entities/dataset.entity'; import { InsecticideResistanceBioassays } from '../insecticideResistance/entities/insecticideResistanceBioassays.entity'; import { InsecticideResistanceService } from '../insecticideResistance/insecticideResistance.service'; @@ -137,7 +138,7 @@ import { DynamicQueryModule } from '../shared/dynamic-query.module'; EditLogsModule, DynamicQueryModule, ], - controllers: [OccurrenceController], + controllers: [OccurrenceController, OccurrenceSearchController], providers: [ OccurrenceService, OccurrenceResolver, diff --git a/src/API/src/db/occurrence/occurrence.resolver.ts b/src/API/src/db/occurrence/occurrence.resolver.ts index 243c85abb..d3127dad6 100644 --- a/src/API/src/db/occurrence/occurrence.resolver.ts +++ b/src/API/src/db/occurrence/occurrence.resolver.ts @@ -25,6 +25,7 @@ import { Reference } from '../shared/entities/reference.entity'; import { ReferenceService } from '../shared/reference.service'; import { flattenOccurrenceRepoObject } from '../../export/utils/allDataCsvCreation'; import { OccurrenceReturn } from './occurrenceReturn'; +import { mapOccurrencesToReturnItems } from './occurrence-response.mapper'; import { randomUUID } from 'crypto'; import { DoiService } from '../doi/doi.service'; import { DoiController } from '../doi/doi.controller'; @@ -164,117 +165,23 @@ export class OccurrenceResolver { recordDownload?: boolean, minimalFields = true, ) { + const effectiveBounds = bounds ?? { locationWindowActive: false }; const { items, total } = await this.occurrenceService.findOccurrences( take, skip, filters, - bounds, + effectiveBounds, minimalFields, ); if (recordDownload) { await this.occurrenceService.incrementDownload(items); } - const relationObject = this.occurrenceService.getOccurrenceFields(true); - const excludeColumns = { - parent: [], - relations: { - // dataset: '*', - dataset: [ - 'id', - 'status', - 'UpdatedBy', - 'UpdatedAt', - 'ReviewedBy', - 'ReviewedAt', - 'ApprovedBy', - 'ApprovedAt', - ], - site: ['longitude_4', 'longitude_5'], - }, - }; - - const includeColumn = ( - isParentProperty: boolean, - relationName: string, - columnName: string, - ) => { - let cols: any; - if (isParentProperty) { - cols = excludeColumns['parent']; - } else { - cols = excludeColumns['relations'][relationName]; - } - - if (cols === '*') return false; - if (Array.isArray(cols) && cols.includes(columnName)) return false; - return true; - }; - - /** - * Include other fields in addition to those specified in the Interface - * extend to other relations. This contradicts strict typing requirements of OccurrenceReturn but it - * was necessary so that we allow inclusion of related fields dynamically - * extended fields will be renamed to `relationName_relationFieldName` - */ - const selectAllFields = (record: object, destinationObject: object) => { - Object.keys(record).map((dataProperty) => { - // check if fld is a relation. If yes, loop through all fields for the relation - if (Object.keys(relationObject).includes(dataProperty)) { - const relationFields = relationObject[dataProperty]; - relationFields.map((relationField) => { - if ( - relationField === 'id' || - !includeColumn(false, dataProperty, relationField) - ) { - // do nothing since field should not be included - } else { - const key = `${dataProperty}_${relationField}`; - Object.assign(destinationObject, { - [key]: record?.[dataProperty]?.[relationField] || null, - }); - } - }); - } else { - if (includeColumn(true, null, dataProperty)) { - Object.assign(destinationObject, { - [dataProperty]: record?.[dataProperty], - }); - } - } - }); - return destinationObject; - }; - - const returnItems: OccurrenceReturn[] = items.map((x) => { - const obj = { - id: x.id, - species: x.recordedSpecies.species, - location: x.site.location, - binary_presence: x.binary_presence, - country: x.site.country, - year_start: x.year_start, - is_adult: !!x.adult_data, - is_larval: !!x.larval_data, - season_val: x.season_calc || x.season_given || '', - insecticide: x.insecticide_resistance_data, - control: x.sample?.control?.toString() || '', - abundance_data: x.abundance_data, - bio_data: x.bio_data, - display_name: x.recordedSpecies?.display_name, - category: x.recordedSpecies?.category, - color: x.recordedSpecies?.color, - //has_bionomics: x.bio_data, - }; - - // extend to other relations. This contradicts strict typing requirements but it - // was necessary so that we allow inclusion of related fields dynamically - if (!minimalFields) { - const extendedObject = selectAllFields(x, obj); - Object.assign(obj, extendedObject); - } - return obj; - }); + const returnItems: OccurrenceReturn[] = mapOccurrencesToReturnItems( + items, + this.occurrenceService, + minimalFields, + ); return Object.assign(new PaginatedOccurrenceData(), { items: returnItems, total, diff --git a/src/API/src/db/occurrence/occurrence.service.ts b/src/API/src/db/occurrence/occurrence.service.ts index 030ee5850..ae721673c 100644 --- a/src/API/src/db/occurrence/occurrence.service.ts +++ b/src/API/src/db/occurrence/occurrence.service.ts @@ -483,57 +483,105 @@ export class OccurrenceService { ); } - if (filters.binary_presence) { + if (filters.binary_presence && filters.binary_presence.length > 0) { + const presenceValues = filters.binary_presence + .filter((val) => val !== null) + .map((val) => val.toUpperCase()); + const includeNull = filters.binary_presence.includes(null); + query = query.andWhere( new Brackets((qb) => { - qb.where( - '"occurrence"."binary_presence" IN (:...binary_presence)', - { - binary_presence: filters.binary_presence, - }, - ); - if (filters.binary_presence.includes(null)) { - qb.orWhere('"occurrence"."binary_presence" IS NULL'); + let hasCondition = false; + if (presenceValues.length > 0) { + qb.where( + 'UPPER("occurrence"."binary_presence") IN (:...binary_presence)', + { binary_presence: presenceValues }, + ); + hasCondition = true; + } + if (includeNull) { + hasCondition + ? qb.orWhere('"occurrence"."binary_presence" IS NULL') + : qb.where('"occurrence"."binary_presence" IS NULL'); } }), ); } - if (filters.abundance_data) { + if (filters.abundance_data && filters.abundance_data.length > 0) { + const abundanceValues = filters.abundance_data + .filter((val) => val !== null) + .map((val) => val.toUpperCase()); + const includeNull = filters.abundance_data.includes(null); + query = query.andWhere( new Brackets((qb) => { - qb.where('"occurrence"."abundance_data" IN (:...abundance_data)', { - abundance_data: filters.abundance_data, - }); - if (filters.abundance_data.includes(null)) { - qb.orWhere('"occurrence"."abundance_data" IS NULL'); + let hasCondition = false; + if (abundanceValues.length > 0) { + qb.where( + 'UPPER("occurrence"."abundance_data") IN (:...abundance_data)', + { abundance_data: abundanceValues }, + ); + hasCondition = true; + } + if (includeNull) { + hasCondition + ? qb.orWhere('"occurrence"."abundance_data" IS NULL') + : qb.where('"occurrence"."abundance_data" IS NULL'); } }), ); } // 2. Repointed isLarval to the occurrence table + // NOTE: larval_data is a free-text column, not boolean. It contains + // 'False'/'FALSE' and 'True'/'TRUE' with inconsistent casing across + // older data and the 2026-06-16 ingestion batch — comparison is done + // case-insensitively via UPPER() on both sides to handle both. if (filters.isLarval && filters.isLarval.length > 0) { + const larvalStringValues = filters.isLarval + .filter((val) => val !== null) + .map((val) => (val ? 'TRUE' : 'FALSE')); + const includeNull = filters.isLarval.includes(null); + query = query.andWhere( new Brackets((qb) => { - qb.where('"occurrence"."larval_data" IN (:...isLarval)', { - isLarval: filters.isLarval, - }); - if (filters.isLarval.includes(null)) { - qb.orWhere('"occurrence"."larval_data" IS NULL'); + let hasCondition = false; + if (larvalStringValues.length > 0) { + qb.where('UPPER("occurrence"."larval_data") IN (:...isLarval)', { + isLarval: larvalStringValues, + }); + hasCondition = true; + } + if (includeNull) { + hasCondition + ? qb.orWhere('"occurrence"."larval_data" IS NULL') + : qb.where('"occurrence"."larval_data" IS NULL'); } }), ); } - // 3. Repointed isAdult to the occurrence table (using abundance_data) + // 3. isAdult filters occurrence.adult_data (TEXT 'True'/'False'/NULL), + // matching what is_adult reads in the response mapper. if (filters.isAdult && filters.isAdult.length > 0) { + const adultStringValues = filters.isAdult + .filter((val) => val !== null) + .map((val) => (val ? 'TRUE' : 'FALSE')); + const includeNull = filters.isAdult.includes(null); + query = query.andWhere( new Brackets((qb) => { - qb.where('"occurrence"."abundance_data" IN (:...isAdult)', { - isAdult: filters.isAdult, - }); - if (filters.isAdult.includes(null)) { - qb.orWhere('"occurrence"."abundance_data" IS NULL'); + let hasCondition = false; + if (adultStringValues.length > 0) { + qb.where('UPPER("occurrence"."adult_data") IN (:...isAdult)', { + isAdult: adultStringValues, + }); + hasCondition = true; + } + if (includeNull) { + hasCondition + ? qb.orWhere('"occurrence"."adult_data" IS NULL') + : qb.where('"occurrence"."adult_data" IS NULL'); } }), ); diff --git a/src/API/src/db/occurrence/search-query.utils.ts b/src/API/src/db/occurrence/search-query.utils.ts new file mode 100644 index 000000000..2aa93fcd0 --- /dev/null +++ b/src/API/src/db/occurrence/search-query.utils.ts @@ -0,0 +1,82 @@ +import { BadRequestException } from '@nestjs/common'; + +export const parseStringArray = (value?: string): string[] | undefined => + value ? value.split(',').map((v) => v.trim()) : undefined; + +export const parseNullableStringArray = ( + value?: string, +): (string | null)[] | undefined => + value + ? value + .split(',') + .map((v) => v.trim()) + .map((v) => (v.toLowerCase() === 'null' ? null : v)) + : undefined; + +export const parseNullableBooleanArray = ( + value?: string, +): (boolean | null)[] | undefined => + value + ? value + .split(',') + .map((v) => v.trim().toLowerCase()) + .map((v) => (v === 'null' ? null : v === 'true')) + : undefined; + +export const parseTimeline = ( + value?: string, +): { startTimestamp?: number; endTimestamp?: number } => { + if (!value) return {}; + const [startYearStr, endYearStr] = value.split('-').map((v) => v.trim()); + const startYear = parseInt(startYearStr, 10); + const endYear = parseInt(endYearStr, 10); + if (isNaN(startYear) || isNaN(endYear)) { + throw new BadRequestException( + `Invalid timeline "${value}". Expected format "YYYY-YYYY", e.g. "2010-2020".`, + ); + } + return { + startTimestamp: Date.UTC(startYear, 0, 1), + endTimestamp: Date.UTC(endYear + 1, 0, 1), + }; +}; + +export const ALLOWED_RESPONSE_FIELDS = [ + 'id', + 'species', + 'location', + 'binary_presence', + 'country', + 'year_start', + 'is_adult', + 'is_larval', + 'season_val', + 'insecticide', + 'control', + 'abundance_data', + 'bio_data', +]; + +export const parseFields = (value?: string): string[] | undefined => { + if (!value) return undefined; + const requested = value.split(',').map((v) => v.trim()); + const invalid = requested.filter((f) => !ALLOWED_RESPONSE_FIELDS.includes(f)); + if (invalid.length > 0) { + throw new BadRequestException( + `Unknown field(s): ${invalid.join( + ', ', + )}. Allowed fields: ${ALLOWED_RESPONSE_FIELDS.join(', ')}.`, + ); + } + return requested; +}; + +/** Always includes `id` even if not explicitly requested, per REST convention. */ +export const projectFields = (item: Record, fields?: string[]) => { + if (!fields) return item; + const projected: Record = { id: item.id }; + fields.forEach((f) => { + projected[f] = item[f]; + }); + return projected; +}; diff --git a/src/API/src/schema.gql b/src/API/src/schema.gql index 527ccee42..4e6648567 100644 --- a/src/API/src/schema.gql +++ b/src/API/src/schema.gql @@ -63,6 +63,17 @@ input Coord { long: Float } +type Country { + alternative_names: [String!]! + creation: DateTime! + id: String! + modified: DateTime! + name: String! + owner: String + sites: [Site!] + updater: String +} + input CreateNewsInput { article: String! id: String @@ -88,8 +99,9 @@ input CreateSpeciesInformationInput { id: String link: String! name: String! + previewImage: String shortDescription: String! - speciesImage: String! + speciesImage: String } """doi""" @@ -198,6 +210,9 @@ type Mutation { deleteSpeciesInformation(id: String!): Boolean! disableNotifications(disable: Boolean!, userId: String!): Boolean! requestRoles(email: String!, requestReason: String!, rolesRequested: [String!]!): Boolean! + updateCountry(input: UpdateCountryInput!): Country! + updateRecordedSpecies(input: UpdateRecordedSpeciesInput!): RecordedSpecies! + updateReference(input: UpdateReferenceInput!, num_id: Int!): Reference! updateUserRoles(input: UserRoleInput!): UserRole! upsertNewsTranslation(input: UpsertNewsTranslationInput!): NewsTranslation! } @@ -323,16 +338,19 @@ type Query { OccurrenceData(bounds: BoundsFilter, filters: OccurrenceFilter, skip: Float = 0, take: Float = 1): PaginatedOccurrenceReturnData! allCommunicationLogs: [CommunicationLog!]! allCommunicationLogsBySentStatus(status: String!): [CommunicationLog!]! + allCountries: [Country!]! allDois: [DOI!]! allDoisByStatus(status: String!): [DOI!]! allGeoData: Bionomics! allNews: [News!]! - allReferenceData(endId: Float = null, order: String = "asc", orderBy: String = "num_id", skip: Float = 0, startId: Float = 1, take: Float = 1, textFilter: String = ""): PaginatedReferenceData! + allRecordedSpecies: [RecordedSpecies!]! + allReferenceData(endId: Float = null, filterField: String = "article_title", order: String = "asc", orderBy: String = "num_id", skip: Float = 0, startId: Float = 1, take: Float = 1, textFilter: String = ""): PaginatedReferenceData! allSpeciesInformation: [SpeciesInformation!]! allUploadedDatasets: [UploadedDataset!] allUploadedModels: [UploadedModel!] allUserRoles: [UserWithRoles!]! communicationLogById(id: String!): CommunicationLog + country(id: String!): Country datasetById(id: String!): Dataset datasets: [Dataset!]! doiById(id: String!): DOI @@ -340,6 +358,7 @@ type Query { getHomepageAnalytics(endAt: Float!, startAt: Float!, timezone: String!, unit: String!): HomepageStats! newsById(id: String!): News! postProcessModel(blobLocation: String!, displayName: String!, maxValue: Float!, modelName: String!, uploadedModelId: String!): ModelProcessingStatus! + recordedSpeciesById(id: String!): RecordedSpecies referenceData(id: String!): Reference! speciesInformationById(id: String!): SpeciesInformation! uploadedDatasetById(id: String!): UploadedDataset @@ -352,6 +371,7 @@ type Query { """recorded species data""" type RecordedSpecies { category: String + color: String display_name: String id: String! species: String! @@ -430,8 +450,33 @@ type SpeciesInformation { id: String! link: String name: String! + previewImage: String shortDescription: String! - speciesImage: String! + speciesImage: String +} + +input UpdateCountryInput { + alternative_names: [String!] + id: String! + name: String +} + +input UpdateRecordedSpeciesInput { + category: String + color: String + displayName: String + id: String! +} + +input UpdateReferenceInput { + article_title: String! + author: String! + citation: String! + journal_title: String! + published: Boolean! + report_type: String! + v_data: Boolean! + year: Float! } """uploaded dataset"""