diff --git a/server/Cargo.toml b/server/Cargo.toml index 868b92193be..19aa6bc6560 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -16,6 +16,8 @@ name = "parsec" crate-type = ["cdylib"] [features] +default = ["pydantic-support"] +pydantic-support = [] # Remember kid: RustCrypto is used if `use-libsodium` is not set ! use-libsodium = ["libparsec_crypto/use-libsodium"] vendored-openssl = ["libparsec_crypto/vendored-openssl"] diff --git a/server/parsec/cli/__init__.py b/server/parsec/cli/__init__.py index d0c9bde1cb9..e5cd601c29a 100644 --- a/server/parsec/cli/__init__.py +++ b/server/parsec/cli/__init__.py @@ -18,6 +18,7 @@ from parsec.cli.sequester_create import create_service, generate_service_certificate from parsec.cli.sequester_list import list_services from parsec.cli.sequester_revoke import generate_service_revocation_certificate, revoke_service +from parsec.cli.tasks import server_tasks_cmd_group from parsec.cli.testbed import TESTBED_AVAILABLE, testbed_cmd __all__ = ("cli",) @@ -53,6 +54,8 @@ def cli() -> None: cli.add_command(list_deletable_realms, "list_deletable_realms") cli.add_command(delete_realm, "delete_realm") cli.add_command(server_sequester_cmd, "sequester") +cli.add_command(server_tasks_cmd_group) + if TESTBED_AVAILABLE: cli.add_command(testbed_cmd, "testbed") # Since `render_email` is only for debugging purpose, we don't expose it diff --git a/server/parsec/cli/tasks/__init__.py b/server/parsec/cli/tasks/__init__.py new file mode 100644 index 00000000000..7d795a43057 --- /dev/null +++ b/server/parsec/cli/tasks/__init__.py @@ -0,0 +1,16 @@ +# Parsec Cloud (https://parsec.cloud) Copyright (c) BUSL-1.1 2016-present Scille SAS + +import click + +from parsec.cli.options import version_option + +from . import list_organization + + +@click.group(name="tasks", short_help="Server tasks collections") +@version_option +def server_tasks_cmd_group() -> None: + pass + + +server_tasks_cmd_group.add_command(list_organization.cmd) diff --git a/server/parsec/cli/tasks/list_organization.py b/server/parsec/cli/tasks/list_organization.py new file mode 100644 index 00000000000..f759c66953a --- /dev/null +++ b/server/parsec/cli/tasks/list_organization.py @@ -0,0 +1,133 @@ +# Parsec Cloud (https://parsec.cloud) Copyright (c) BUSL-1.1 2016-present Scille SAS +from __future__ import annotations + +from collections.abc import AsyncGenerator +from contextlib import asynccontextmanager + +import click +import pydantic + +from parsec._parsec import ( + ActiveUsersLimit, + DateTime, + EmailAddress, + OrganizationID, + ParsecAddr, + SecretKey, +) +from parsec.cli.options import asyncio_run, db_server_options, logging_config_options +from parsec.components.memory.datamodel import MemoryDatamodel +from parsec.components.memory.events import event_bus_factory +from parsec.components.memory.organization import MemoryOrganizationComponent +from parsec.components.organization import ( + BaseOrganizationComponent, + OrganizationDump, + TermsOfService, +) +from parsec.components.postgresql.handler import asyncpg_pool_factory +from parsec.components.postgresql.organization import PGOrganizationComponent +from parsec.config import ( + BackendConfig, + BaseDatabaseConfig, + LogLevel, + MockedBlockStoreConfig, + MockedEmailConfig, + PostgreSQLDatabaseConfig, +) +from parsec.webhooks import MockedWebhooksComponent + + +@pydantic.dataclasses.dataclass() +class OrganizationInfo: + id: OrganizationID + created_on: DateTime + is_bootstrapped: bool + bootstrapped_on: DateTime | None + is_expired: bool + expired_on: DateTime | None + active_users_limit: ActiveUsersLimit + user_profile_outsider_allowed: bool + realm_minimum_archiving_period_before_deletion: int + tos: TermsOfService | None + + @classmethod + def from_dump(cls, dump: OrganizationDump) -> OrganizationInfo: + return OrganizationInfo( + id=dump.organization_id, + created_on=dump.created_on, + bootstrapped_on=dump.bootstrapped_on, + is_bootstrapped=dump.is_bootstrapped, + expired_on=dump.expired_on, + is_expired=dump.is_expired, + user_profile_outsider_allowed=dump.user_profile_outsider_allowed, + active_users_limit=dump.active_users_limit, + realm_minimum_archiving_period_before_deletion=dump.realm_minimum_archiving_period_before_deletion, + tos=dump.tos, + ) + + +@click.command(name="list-organizations", short_help="List organization present on the server") +@db_server_options +@logging_config_options(default_log_level="INFO") +@asyncio_run +async def cmd( + db: BaseDatabaseConfig, + db_min_connections: int, + db_max_connections: int, + log_level: LogLevel, + log_format: str, + log_file: str | None, +): + backend_config = BackendConfig( + debug=False, + db_config=db, + blockstore_config=MockedBlockStoreConfig(), + email_config=MockedEmailConfig(sender=EmailAddress("tasks@parsec.local")), + server_addr=ParsecAddr("tasks.parsec.local", None, True), + administration_token="", + fake_account_password_algorithm_seed=SecretKey.generate(), + ) + async with organization_component_factory(backend_config) as component: + orgs = await list_organizations(component) + + adapter = pydantic.TypeAdapter( + dict[ + OrganizationID, + OrganizationInfo, + ] + ) + click.echo_via_pager(adapter.dump_json(orgs, indent=4).decode()) + + +@asynccontextmanager +async def organization_component_factory( + config: BackendConfig, +) -> AsyncGenerator[BaseOrganizationComponent, None]: + if config.db_config.is_mocked(): + data = MemoryDatamodel( + {} if config.backend_mocked_data is None else config.backend_mocked_data + ) + async with event_bus_factory() as event_bus: + yield MemoryOrganizationComponent(data, event_bus, MockedWebhooksComponent(), config) + + else: + assert isinstance(config.db_config, PostgreSQLDatabaseConfig) + async with asyncpg_pool_factory( + url=config.db_config.url, + min_connections=config.db_config.min_connections, + max_connections=config.db_config.max_connections, + ) as pool: + yield PGOrganizationComponent( + pool=pool, + webhooks=MockedWebhooksComponent(), + config=None, # pyright: ignore [reportArgumentType] + ) + + +async def list_organizations( + component: BaseOrganizationComponent, +) -> dict[OrganizationID, OrganizationInfo]: + return { + id: OrganizationInfo.from_dump(dump) + for id, dump in (await component.list_organizations()).items() + } diff --git a/server/parsec/components/memory/datamodel.py b/server/parsec/components/memory/datamodel.py index cefa907189e..bd9a78cca9d 100644 --- a/server/parsec/components/memory/datamodel.py +++ b/server/parsec/components/memory/datamodel.py @@ -152,6 +152,7 @@ class MemoryOrganization: sequester_authority_certificate: bytes | None = field(default=None, repr=False) cooked_sequester_authority: SequesterAuthorityCertificate | None = None root_verify_key: VerifyKey | None = field(default=None, repr=False) + expired_on: DateTime | None = None is_expired: bool = False realm_minimum_archiving_period_before_deletion: int = 2592000 # 30 days tos: TermsOfService | None = None diff --git a/server/parsec/components/memory/organization.py b/server/parsec/components/memory/organization.py index 0851f19163f..b87c0aece53 100644 --- a/server/parsec/components/memory/organization.py +++ b/server/parsec/components/memory/organization.py @@ -135,6 +135,7 @@ async def get(self, id: OrganizationID) -> Organization | OrganizationGetBadOutc organization_id=org.organization_id, bootstrap_token=org.bootstrap_token, is_expired=org.is_expired, + expired_on=org.expired_on, created_on=org.created_on, bootstrapped_on=org.bootstrapped_on, root_verify_key=org.root_verify_key, @@ -366,6 +367,7 @@ async def update( if is_expired is not Unset: org.is_expired = is_expired + org.expired_on = now if is_expired else None if active_users_limit is not Unset: org.active_users_limit = active_users_limit if user_profile_outsider_allowed is not Unset: @@ -402,7 +404,7 @@ async def get_tos( return org.tos @override - async def test_dump_organizations( + async def list_organizations( self, skip_templates: bool = True ) -> dict[OrganizationID, OrganizationDump]: items = {} @@ -413,8 +415,11 @@ async def test_dump_organizations( org.active_users_limit items[org.organization_id] = OrganizationDump( organization_id=org.organization_id, + created_on=org.created_on, bootstrap_token=org.bootstrap_token, + bootstrapped_on=org.bootstrapped_on, is_bootstrapped=org.is_bootstrapped, + expired_on=org.expired_on, is_expired=org.is_expired, active_users_limit=org.active_users_limit, user_profile_outsider_allowed=org.user_profile_outsider_allowed, diff --git a/server/parsec/components/organization.py b/server/parsec/components/organization.py index b1040e4d79d..a2cdb387326 100644 --- a/server/parsec/components/organization.py +++ b/server/parsec/components/organization.py @@ -60,8 +60,11 @@ class TermsOfService: @dataclass(slots=True) class OrganizationDump: organization_id: OrganizationID + created_on: DateTime bootstrap_token: AccessToken | None + bootstrapped_on: DateTime | None is_bootstrapped: bool + expired_on: DateTime | None is_expired: bool active_users_limit: ActiveUsersLimit user_profile_outsider_allowed: bool @@ -171,6 +174,7 @@ class Organization: organization_id: OrganizationID bootstrap_token: AccessToken | None is_expired: bool + expired_on: DateTime | None created_on: DateTime realm_minimum_archiving_period_before_deletion: int bootstrapped_on: DateTime | None @@ -321,7 +325,7 @@ async def get_tos( ) -> TermsOfService | OrganizationGetTosBadOutcome | None: raise NotImplementedError - async def test_dump_organizations( + async def list_organizations( self, skip_templates: bool = True ) -> dict[OrganizationID, OrganizationDump]: raise NotImplementedError diff --git a/server/parsec/components/postgresql/organization.py b/server/parsec/components/postgresql/organization.py index 6b701230132..2326c889165 100644 --- a/server/parsec/components/postgresql/organization.py +++ b/server/parsec/components/postgresql/organization.py @@ -38,13 +38,13 @@ from parsec.components.postgresql.organization_bootstrap import organization_bootstrap from parsec.components.postgresql.organization_create import organization_create from parsec.components.postgresql.organization_get_tos import organization_get_tos +from parsec.components.postgresql.organization_list_organizations import ( + organization_list_organizations, +) from parsec.components.postgresql.organization_stats import ( organization_server_stats, organization_stats, ) -from parsec.components.postgresql.organization_test_dump_organizations import ( - organization_test_dump_organizations, -) from parsec.components.postgresql.organization_test_dump_topics import organization_test_dump_topics from parsec.components.postgresql.organization_update import organization_update from parsec.components.postgresql.test_queries import ( @@ -69,6 +69,7 @@ def _make_q_get_organization(for_update: bool = False) -> Q: bootstrap_token, root_verify_key, is_expired, + _expired_on AS expired_on, _bootstrapped_on AS bootstrapped_on, _created_on AS created_on, active_users_limit, @@ -247,6 +248,7 @@ async def _get( bootstrap_token=bootstrap_token, root_verify_key=rvk, is_expired=row["is_expired"], + expired_on=row["expired_on"], created_on=row["created_on"], bootstrapped_on=row["bootstrapped_on"], active_users_limit=ActiveUsersLimit.from_maybe_int(row["active_users_limit"]), @@ -355,10 +357,10 @@ async def get_tos( @override @no_transaction - async def test_dump_organizations( + async def list_organizations( self, conn: AsyncpgConnection, skip_templates: bool = True ) -> dict[OrganizationID, OrganizationDump]: - return await organization_test_dump_organizations(conn, skip_templates) + return await organization_list_organizations(conn, skip_templates) @override @no_transaction diff --git a/server/parsec/components/postgresql/organization_test_dump_organizations.py b/server/parsec/components/postgresql/organization_list_organizations.py similarity index 81% rename from server/parsec/components/postgresql/organization_test_dump_organizations.py rename to server/parsec/components/postgresql/organization_list_organizations.py index fa7661dd172..f0ce2c0245d 100644 --- a/server/parsec/components/postgresql/organization_test_dump_organizations.py +++ b/server/parsec/components/postgresql/organization_list_organizations.py @@ -19,8 +19,11 @@ _q_get_organizations = Q(""" SELECT organization_id, + _created_on AS created_on, bootstrap_token, + _bootstrapped_on AS bootstrapped_on, is_expired, + _expired_on AS expired_on, active_users_limit, user_profile_outsider_allowed, realm_minimum_archiving_period_before_deletion, @@ -32,7 +35,7 @@ """) -async def organization_test_dump_organizations( +async def organization_list_organizations( conn: AsyncpgConnection, skip_templates: bool = True ) -> dict[OrganizationID, OrganizationDump]: items = {} @@ -48,6 +51,12 @@ async def organization_test_dump_organizations( if skip_templates and organization_id.str.endswith("Template"): continue + match row["created_on"]: + case DateTime() as created_on: + pass + case _: + assert False, row + match row["bootstrap_token"]: case str() as raw_bootstrap_token: bootstrap_token = AccessToken.from_hex(raw_bootstrap_token) @@ -62,12 +71,28 @@ async def organization_test_dump_organizations( case _: assert False, row + match row["bootstrapped_on"]: + case DateTime() as bootstrapped_on: + pass + case None: + bootstrapped_on = None + case _: + assert False, row + match row["is_expired"]: case bool() as is_expired: pass case _: assert False, row + match row["expired_on"]: + case DateTime() as expired_on: + pass + case None: + expired_on = None + case _: + assert False, row + match row["active_users_limit"]: case None: active_users_limit = ActiveUsersLimit.NO_LIMIT @@ -103,9 +128,12 @@ async def organization_test_dump_organizations( items[organization_id] = OrganizationDump( organization_id=organization_id, + created_on=created_on, bootstrap_token=bootstrap_token, is_bootstrapped=is_bootstrapped, + bootstrapped_on=bootstrapped_on, is_expired=is_expired, + expired_on=expired_on, active_users_limit=active_users_limit, user_profile_outsider_allowed=user_profile_outsider_allowed, realm_minimum_archiving_period_before_deletion=realm_minimum_archiving_period_before_deletion, diff --git a/server/parsec/webhooks.py b/server/parsec/webhooks.py index 9d8a4281641..3a41ff63d3b 100644 --- a/server/parsec/webhooks.py +++ b/server/parsec/webhooks.py @@ -236,3 +236,55 @@ async def sequester_service_on_realm_rotate_key( return SequesterServiceUnavailable( service_id=service_id, ) + + +class MockedWebhooksComponent(WebhooksComponent): + """ + Webhooks that does nothing, + + Used when creating components that require a webhook class but we do not except that component to use webhook (useful for tasks)""" + + def __init__(self) -> None: + pass + + async def on_organization_bootstrap( + self, + organization_id: OrganizationID, + device_id: DeviceID, + device_label: DeviceLabel, + human_email: EmailAddress, + human_label: str, + ) -> None: + return None + + async def sequester_service_on_vlob_create_or_update( + self, + webhook_url: str, + service_id: SequesterServiceID, + organization_id: OrganizationID, + author: DeviceID, + realm_id: VlobID, + vlob_id: VlobID, + key_index: int, + version: int, + timestamp: DateTime, + blob: bytes, + ) -> SequesterServiceUnavailable | RejectedBySequesterService | None: + return None + + async def sequester_service_on_realm_rotate_key( + self, + webhook_url: str, + service_id: SequesterServiceID, + organization_id: OrganizationID, + keys_bundle: bytes, + keys_bundle_access: bytes, + author: DeviceID, + timestamp: DateTime, + realm_id: VlobID, + key_index: int, + encryption_algorithm: SecretKeyAlgorithm, + hash_algorithm: HashAlgorithm, + key_canary: bytes, + ) -> SequesterServiceUnavailable | RejectedBySequesterService | None: + return None diff --git a/server/src/ids.rs b/server/src/ids.rs index db977a39498..6475518a90e 100644 --- a/server/src/ids.rs +++ b/server/src/ids.rs @@ -2,7 +2,11 @@ use std::str::FromStr; -use pyo3::{exceptions::PyValueError, prelude::*, types::PyType}; +use pyo3::{ + exceptions::PyValueError, + prelude::*, + types::{PyType, PyTypeMethods}, +}; // UUID based type @@ -203,7 +207,12 @@ impl OrganizationID { Err(err) => Err(PyValueError::new_err(err.to_string())), } } else { - Err(PyValueError::new_err("Unimplemented")) + // NOTE: We return `ValueError` instead of `TypeError` to be able to use it with + // pydantic `PlainValidator` + Err(PyValueError::new_err(format!( + "Does not support converting {} to OrganizationID", + organization_id.get_type().name()?.to_str()? + ))) } } @@ -211,6 +220,37 @@ impl OrganizationID { fn str(&self) -> &str { self.0.as_ref() } + + #[classmethod] + #[pyo3(name = "__get_pydantic_core_schema__")] + fn get_pydantic_core_schema<'py>( + cls: &Bound<'py, PyType>, + _source_type: &Bound<'_, PyType>, + _handler: &Bound<'_, PyAny>, + py: Python<'py>, + ) -> PyResult> { + use crate::pydantic_support::inner::CoreSchemaModule; + let core_schema = CoreSchemaModule::new(py)?; + + let str_schema = core_schema.str_schema()?; + + // Create a string schema to load the value from string. + let from_str = core_schema.no_info_after_validator_function( + cls, + str_schema, + // Indicate to pydantic that it just need to call `str(val)` to serialize the value + Some(core_schema.to_string_ser_schema()?), + py, + )?; + + // Make pydantic accept own class as value + let instance_schema = core_schema.instance_schema(cls)?; + + // Build schema that accept both string or instance value. + let union_schema = core_schema.union_schema([instance_schema, from_str], py)?; + + union_schema.into_pyobject(py).map_err(Into::into) + } } crate::binding_utils::gen_py_wrapper_class_for_id!( diff --git a/server/src/lib.rs b/server/src/lib.rs index 426a3a860bb..7a6fc7b7eaf 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -15,6 +15,7 @@ mod enumerate; mod ids; mod misc; mod protocol; +pub(crate) mod pydantic_support; #[cfg(feature = "test-utils")] mod testbed; mod time; diff --git a/server/src/protocol.rs b/server/src/protocol.rs index b1186da43ec..7294a3ab7ed 100644 --- a/server/src/protocol.rs +++ b/server/src/protocol.rs @@ -4,7 +4,7 @@ use pyo3::{ exceptions::PyValueError, prelude::{PyAnyMethods, PyModuleMethods}, pyclass, pymethods, - types::{PyInt, PyModule, PyType}, + types::{PyDict, PyInt, PyList, PyModule, PyType}, Bound, IntoPyObjectExt, Py, PyAny, PyResult, Python, }; @@ -73,6 +73,47 @@ impl ActiveUsersLimit { libparsec_types::ActiveUsersLimit::NoLimit => None, } } + + #[classmethod] + #[pyo3(name = "__get_pydantic_core_schema__")] + fn get_pydantic_core_schema<'py>( + cls: &Bound<'py, PyType>, + _source_type: &Bound<'_, PyType>, + _handler: &Bound<'_, PyAny>, + py: Python<'py>, + ) -> PyResult> { + let core_schema = py.import("pydantic_core")?.getattr("core_schema")?; + + let int_schema = core_schema.call_method0("int_schema")?; + let nullable_int_schema = core_schema.call_method1("nullable_schema", (int_schema,))?; + + let validator_kwargs = PyDict::new(py); + + let ser_kwargs = PyDict::new(py); + ser_kwargs.set_item("return_schema", nullable_int_schema.clone())?; + ser_kwargs.set_item("when_used", "always")?; + let ser_schema = core_schema.call_method( + "plain_serializer_function_ser_schema", + (cls.getattr("to_maybe_int")?,), + Some(&ser_kwargs), + )?; + validator_kwargs.set_item("serialization", ser_schema)?; + + let int_validator = core_schema.call_method( + "no_info_after_validator_function", + (cls.getattr("from_maybe_int")?, nullable_int_schema), + Some(&validator_kwargs), + )?; + + let instance_schema = core_schema.call_method1("is_instance_schema", (cls,))?; + + let union_schema = core_schema.call_method1( + "union_schema", + (PyList::new(py, vec![instance_schema, int_validator])?,), + )?; + + Ok(union_schema) + } } python_bindings_parsec_protocol_cmds_family!("../libparsec/crates/protocol/schema/anonymous_cmds"); diff --git a/server/src/pydantic_support.rs b/server/src/pydantic_support.rs new file mode 100644 index 00000000000..37c71688b40 --- /dev/null +++ b/server/src/pydantic_support.rs @@ -0,0 +1,166 @@ +// Parsec Cloud (https://parsec.cloud) Copyright (c) BUSL-1.1 2016-present Scille SAS + +macro_rules! pydantic_json_schema { + ($class:ident, $($key:ident=$value:expr),+) => { + #[cfg(feature = "pydantic-support")] + #[pymethods] + impl $class { + #[classmethod] + #[pyo3(name = "__get_pydantic_json_schema__")] + fn get_pydantic_json_schema<'py>( + _cls: &::pyo3::Bound<'_, ::pyo3::types::PyType>, + _schema: &::pyo3::Bound<'_, ::pyo3::types::PyAny>, + _handler: &::pyo3::Bound<'_, ::pyo3::types::PyAny>, + py: ::pyo3::Python<'py>, + ) -> ::pyo3::PyResult<::pyo3::Bound<'py, ::pyo3::types::PyDict>> { + let dict = ::pyo3::types::PyDict::new(py); + $( + dict.set_item(stringify!($key), $value)?; + )+ + Ok(dict) + } + } + }; +} + +pub(crate) use pydantic_json_schema; + +#[cfg(feature = "pydantic-support")] +pub mod inner { + use pyo3::{ + conversion::IntoPyObject, + types::{PyAny, PyAnyMethods, PyDict, PyList, PyType}, + Bound, PyResult, Python, + }; + + pub struct CoreSchemaModule<'py>(Bound<'py, PyAny>); + + pub type AnyItem<'py> = Bound<'py, PyAny>; + + impl<'py> CoreSchemaModule<'py> { + pub fn new(py: Python<'py>) -> PyResult { + py.import("pydantic_core") + .and_then(|module| module.getattr("core_schema")) + .map(Self) + } + + pub fn str_schema(&self) -> PyResult> { + self.0.call_method0("str_schema").map(CoreSchema) + } + + pub fn to_string_ser_schema(&self) -> PyResult> { + self.0.call_method0("to_string_ser_schema").map(CoreSchema) + } + + pub fn instance_schema(&self, cls: &Bound<'py, PyType>) -> PyResult> { + self.0 + .call_method1("is_instance_schema", (cls,)) + .map(CoreSchema) + } + + pub fn union_schema( + &self, + schemas: impl IntoIterator, + py: Python<'py>, + ) -> PyResult> + where + T: IntoPyObject<'py>, + { + self.0 + .call_method1("union_schema", (PyList::new(py, schemas)?,)) + .map(CoreSchema) + } + + pub fn plain_serializer_function_ser_schema( + &self, + func: impl IntoPyObject<'py>, + return_schema: Option>, + when_used: Option, + py: Python<'py>, + ) -> PyResult> { + let kwargs = PyDict::new(py); + if let Some(return_schema) = return_schema { + kwargs.set_item("return_schema", return_schema)?; + } + if let Some(when_used) = when_used { + kwargs.set_item("when_used", when_used.as_str())?; + } + + self.0 + .call_method( + "plain_serializer_function_ser_schema", + (func,), + Some(&kwargs), + ) + .map(CoreSchema) + } + + pub fn no_info_after_validator_function( + &self, + func: impl IntoPyObject<'py>, + schema: CoreSchema<'py>, + serialization: Option>, + py: Python<'py>, + ) -> PyResult> { + let kwargs = if let Some(serialization) = serialization { + let kwargs = PyDict::new(py); + kwargs.set_item("serialization", serialization)?; + Some(kwargs) + } else { + None + }; + self.0 + .call_method( + "no_info_after_validator_function", + (func, schema), + kwargs.as_ref(), + ) + .map(CoreSchema) + } + } + + #[derive(Clone)] + pub struct CoreSchema<'py>(AnyItem<'py>); + + impl<'py> From> for AnyItem<'py> { + fn from(value: CoreSchema<'py>) -> Self { + value.0 + } + } + + impl<'py> IntoPyObject<'py> for CoreSchema<'py> { + type Target = PyAny; + + type Output = Bound<'py, Self::Target>; + + type Error = std::convert::Infallible; + + fn into_pyobject(self, _py: Python<'py>) -> Result { + Ok(self.0) + } + } + + #[derive(Default, Clone, Copy, PartialEq, Eq)] + pub enum WhenUsed { + /// Means always use + #[default] + Always, + /// Use unless the value is None + UnlessNone, + /// Use when serializing to JSON + Json, + /// Use when serializing to JSON and the value is not None + JsonUnlessNone, + } + + impl WhenUsed { + pub const fn as_str(&self) -> &'static str { + match self { + WhenUsed::Always => "always", + WhenUsed::UnlessNone => "unless-none", + WhenUsed::Json => "json", + WhenUsed::JsonUnlessNone => "json-unless-none", + } + } + } +} diff --git a/server/src/time.rs b/server/src/time.rs index 4a7bcee9b75..eaf34e9e608 100644 --- a/server/src/time.rs +++ b/server/src/time.rs @@ -152,4 +152,45 @@ impl DateTime { + microseconds as i64; Ok(Self(self.0.add_us(us))) } + + #[classmethod] + #[cfg(feature = "pydantic-support")] + #[pyo3(name = "__get_pydantic_core_schema__")] + fn get_pydantic_core_schema<'py>( + cls: &Bound<'py, PyType>, + _source_type: &Bound<'_, PyType>, + _handler: &Bound<'_, PyAny>, + py: Python<'py>, + ) -> PyResult> { + use crate::pydantic_support::inner::{CoreSchemaModule, WhenUsed}; + + let core_schema = CoreSchemaModule::new(py)?; + + let str_schema = core_schema.str_schema()?; + + // Serialize into string using `Self::to_rfc3339` + let ser_schema = core_schema.plain_serializer_function_ser_schema( + cls.getattr("to_rfc3339")?, + Some(str_schema.clone()), + Some(WhenUsed::Always), + py, + )?; + + // Validate string using `Self::from_rfc3339` + let str_validator = core_schema.no_info_after_validator_function( + cls.getattr("from_rfc3339")?, + str_schema, + Some(ser_schema), + py, + )?; + + let instance_validator = core_schema.instance_schema(cls)?; + + // Support both instance and string schema + let union_validator = core_schema.union_schema([instance_validator, str_validator], py)?; + + union_validator.into_pyobject(py).map_err(Into::into) + } } + +crate::pydantic_support::pydantic_json_schema!(DateTime, type="string", format="date-time"); diff --git a/server/tests/administration/test_create_organization.py b/server/tests/administration/test_create_organization.py index 2e3eb4aae19..0bc4dddf6b9 100644 --- a/server/tests/administration/test_create_organization.py +++ b/server/tests/administration/test_create_organization.py @@ -15,6 +15,7 @@ ParsecOrganizationBootstrapAddr, ) from parsec.components.organization import ( + Organization, OrganizationDump, TermsOfService, UnsetType, @@ -198,12 +199,17 @@ async def test_ok( case tos: expected_tos = TermsOfService(updated_on=ANY, per_locale_urls=tos) - dump = await backend.organization.test_dump_organizations() + dump = await backend.organization.list_organizations() + org = await backend.organization.get(org_id) + assert isinstance(org, Organization) assert dump[org_id] == OrganizationDump( organization_id=org_id, + created_on=org.created_on, bootstrap_token=bootstrap_token, is_bootstrapped=False, + bootstrapped_on=None, is_expired=False, + expired_on=None, active_users_limit=expected_active_users_limit, user_profile_outsider_allowed=expected_user_profile_outsider_allowed, realm_minimum_archiving_period_before_deletion=expected_minimum_archiving_period, @@ -231,12 +237,17 @@ async def test_overwrite_existing( assert isinstance(outcome, AccessToken) # Sanity check - dump = await backend.organization.test_dump_organizations() + dump = await backend.organization.list_organizations() + org = await backend.organization.get(org_id) + assert isinstance(org, Organization) assert dump[org_id] == OrganizationDump( organization_id=org_id, + created_on=org.created_on, bootstrap_token=bootstrap_token, is_bootstrapped=False, + bootstrapped_on=None, is_expired=False, + expired_on=None, active_users_limit=ActiveUsersLimit.limited_to(1), user_profile_outsider_allowed=False, realm_minimum_archiving_period_before_deletion=2, @@ -262,12 +273,17 @@ async def test_overwrite_existing( new_bootstrap_token = ParsecOrganizationBootstrapAddr.from_url(body["bootstrap_url"]).token assert new_bootstrap_token != bootstrap_token.hex - dump = await backend.organization.test_dump_organizations() + dump = await backend.organization.list_organizations() + org = await backend.organization.get(org_id) + assert isinstance(org, Organization) assert dump[org_id] == OrganizationDump( organization_id=org_id, + created_on=org.created_on, bootstrap_token=new_bootstrap_token, is_bootstrapped=False, + bootstrapped_on=None, is_expired=False, + expired_on=None, active_users_limit=ActiveUsersLimit.NO_LIMIT, user_profile_outsider_allowed=True, realm_minimum_archiving_period_before_deletion=1000, diff --git a/server/tests/administration/test_patch_organization.py b/server/tests/administration/test_patch_organization.py index faa2e1db2d3..30891faee10 100644 --- a/server/tests/administration/test_patch_organization.py +++ b/server/tests/administration/test_patch_organization.py @@ -7,7 +7,13 @@ import pytest from parsec._parsec import ActiveUsersLimit -from parsec.components.organization import OrganizationDump, TermsOfService, TosLocale, TosUrl +from parsec.components.organization import ( + Organization, + OrganizationDump, + TermsOfService, + TosLocale, + TosUrl, +) from parsec.events import EventOrganizationExpired, EventOrganizationTosUpdated from tests.common import AdminUnauthErrorsTester, Backend, CoolorgRpcClients @@ -126,12 +132,21 @@ async def test_ok( assert response.status_code == 200, response.content assert response.json() == {} - dump = await backend.organization.test_dump_organizations() + dump = await backend.organization.list_organizations() + org = await backend.organization.get(coolorg.organization_id) + assert isinstance(org, Organization) + is_expired = params.get("is_expired", False) + assert (is_expired and org.expired_on is not None) or ( + not is_expired and org.expired_on is None + ) assert dump[coolorg.organization_id] == OrganizationDump( organization_id=coolorg.organization_id, + created_on=org.created_on, bootstrap_token=ANY, is_bootstrapped=True, - is_expired=params.get("is_expired", False), + bootstrapped_on=org.bootstrapped_on, + is_expired=is_expired, + expired_on=org.expired_on, active_users_limit=ActiveUsersLimit.from_maybe_int(params.get("active_users_limit", None)), user_profile_outsider_allowed=params.get("user_profile_outsider_allowed", True), realm_minimum_archiving_period_before_deletion=params.get( @@ -164,7 +179,7 @@ async def test_expire_and_cancel_expire( EventOrganizationExpired(organization_id=coolorg.organization_id) ) - dump = await backend.organization.test_dump_organizations() + dump = await backend.organization.list_organizations() assert dump[coolorg.organization_id].is_expired is True # Re-expire, should be a no-op @@ -182,7 +197,7 @@ async def test_expire_and_cancel_expire( EventOrganizationExpired(organization_id=coolorg.organization_id) ) - dump = await backend.organization.test_dump_organizations() + dump = await backend.organization.list_organizations() assert dump[coolorg.organization_id].is_expired is True # Cancel expiration @@ -196,7 +211,7 @@ async def test_expire_and_cancel_expire( # Cancelling the expiration doesn't trigger any event - dump = await backend.organization.test_dump_organizations() + dump = await backend.organization.list_organizations() assert dump[coolorg.organization_id].is_expired is False # Re-cancel expiration, should be a no-op @@ -210,7 +225,7 @@ async def test_expire_and_cancel_expire( # Cancelling the expiration doesn't trigger any event - dump = await backend.organization.test_dump_organizations() + dump = await backend.organization.list_organizations() assert dump[coolorg.organization_id].is_expired is False @@ -235,7 +250,7 @@ async def test_set_unset_tos( EventOrganizationTosUpdated(organization_id=coolorg.organization_id) ) - dump = await backend.organization.test_dump_organizations() + dump = await backend.organization.list_organizations() assert dump[coolorg.organization_id].tos == TermsOfService( updated_on=ANY, per_locale_urls={"fr_CA": "https://parsec.invalid/tos_fr1"} ) @@ -258,7 +273,7 @@ async def test_set_unset_tos( EventOrganizationTosUpdated(organization_id=coolorg.organization_id) ) - dump = await backend.organization.test_dump_organizations() + dump = await backend.organization.list_organizations() assert dump[coolorg.organization_id].tos == TermsOfService( updated_on=ANY, per_locale_urls={ @@ -280,7 +295,7 @@ async def test_set_unset_tos( EventOrganizationTosUpdated(organization_id=coolorg.organization_id) ) - dump = await backend.organization.test_dump_organizations() + dump = await backend.organization.list_organizations() assert dump[coolorg.organization_id].tos is None diff --git a/server/tests/api_v5/anonymous/test_organization_bootstrap.py b/server/tests/api_v5/anonymous/test_organization_bootstrap.py index 74d1efc9ddd..61af0460fa4 100644 --- a/server/tests/api_v5/anonymous/test_organization_bootstrap.py +++ b/server/tests/api_v5/anonymous/test_organization_bootstrap.py @@ -133,7 +133,7 @@ async def test_anonymous_organization_bootstrap_ok( # Ensure the default config has been used to configure the organization # when spontaneously created. if config.spontaneous: - orgs = await backend.organization.test_dump_organizations() + orgs = await backend.organization.list_organizations() org = orgs[organization_id] assert org.active_users_limit == backend.config.organization_initial_active_users_limit assert ( diff --git a/server/tests/cli/tasks/test_list_organizations.py b/server/tests/cli/tasks/test_list_organizations.py new file mode 100644 index 00000000000..7f13aee71ea --- /dev/null +++ b/server/tests/cli/tasks/test_list_organizations.py @@ -0,0 +1,76 @@ +# Parsec Cloud (https://parsec.cloud) Copyright (c) BUSL-1.1 2016-present Scille SAS + +import json + +from click.testing import CliRunner +from pydantic import TypeAdapter + +from parsec._parsec import ActiveUsersLimit, DateTime +from parsec.cli.tasks import list_organization +from parsec.cli.testbed import TestbedBackend +from parsec.components.organization import Organization +from parsec.config import BaseDatabaseConfig, PostgreSQLDatabaseConfig +from tests.common.client import CoolorgRpcClients + + +def test_list_organization_cmd(db_config: BaseDatabaseConfig, db_args: list[str]): + runner = CliRunner() + args = db_args + use_pg = isinstance(db_config, PostgreSQLDatabaseConfig) + result = runner.invoke(list_organization.cmd, args) + assert result.exception is None, result.exc_info + assert result.exit_code == 0 + assert result.stderr_bytes == b"" + data = json.loads(result.stdout) + if not use_pg: + assert data == {} + else: + assert isinstance(data, dict) + assert list(data.keys()) != [] + + +async def test_list_organization(coolorg: CoolorgRpcClients, testbed: TestbedBackend): + orgs = await list_organization.list_organizations(testbed.backend.organization) + coolorg_org = await testbed.backend.organization.get(coolorg.organization_id) + assert isinstance(coolorg_org, Organization) + + assert coolorg.organization_id in orgs + assert orgs[coolorg.organization_id] == list_organization.OrganizationInfo( + id=coolorg.organization_id, + created_on=coolorg_org.created_on, + bootstrapped_on=coolorg_org.bootstrapped_on, + is_bootstrapped=True, + expired_on=coolorg_org.expired_on, + is_expired=False, + user_profile_outsider_allowed=True, + realm_minimum_archiving_period_before_deletion=testbed.backend.config.organization_initial_realm_deletion_min_archiving_period, + active_users_limit=coolorg_org.active_users_limit, + tos=None, + ) + + +async def test_serialization(coolorg: CoolorgRpcClients): + now = DateTime.now() + info = list_organization.OrganizationInfo( + id=coolorg.organization_id, + created_on=now, + bootstrapped_on=now, + expired_on=None, + is_expired=False, + is_bootstrapped=True, + user_profile_outsider_allowed=True, + realm_minimum_archiving_period_before_deletion=5, + active_users_limit=ActiveUsersLimit.limited_to(5), + tos=None, + ) + + adapter = TypeAdapter(list_organization.OrganizationInfo) + serialized = adapter.dump_json(info) + raw_info = json.loads(serialized) + print(raw_info) + assert raw_info["id"] == info.id.str + assert raw_info["bootstrapped_on"] == now.to_rfc3339() + assert raw_info["active_users_limit"] == 5 + + got = adapter.validate_json(serialized) + assert got == info diff --git a/server/tests/conftest.py b/server/tests/conftest.py index 58c3c3a1849..033b9ee0e51 100644 --- a/server/tests/conftest.py +++ b/server/tests/conftest.py @@ -279,6 +279,18 @@ async def db_config(request: pytest.FixtureRequest) -> BaseDatabaseConfig: return MockedDatabaseConfig() +@pytest.fixture +async def db_args(db_config: BaseDatabaseConfig) -> list[str]: + if isinstance(db_config, PostgreSQLDatabaseConfig): + return [ + f"--db={db_config.url}", + f"--db-min-connections={db_config.min_connections}", + f"--db-max-connections={db_config.max_connections}", + ] + else: + return ["--db=MOCKED"] + + @pytest.fixture def blockstore_config(db_config: BaseDatabaseConfig) -> BaseBlockStoreConfig: # TODO: allow to test against swift ?