From f9c8c46cd82a453674398e1fc739a7974c44c5e6 Mon Sep 17 00:00:00 2001 From: JosepSampe Date: Mon, 19 Oct 2026 18:20:37 +0200 Subject: [PATCH 01/13] Fix backends --- CHANGELOG.md | 7 +- config/config_template.yaml | 13 +- docs/source/storage_config/swift.md | 34 +- lithops/executors.py | 65 +++- lithops/retries.py | 45 ++- .../backends/gcp_cloudrun/cloudrun.py | 48 ++- .../backends/gcp_functions/gcp_functions.py | 31 +- lithops/serverless/backends/k8s/k8s.py | 7 +- .../serverless/backends/knative/knative.py | 23 +- .../standalone/backends/aws_ec2/aws_ec2.py | 12 +- .../standalone/backends/ibm_vpc/ibm_vpc.py | 13 + lithops/storage/backends/aws_s3/aws_s3.py | 6 +- .../backends/azure_storage/azure_storage.py | 13 +- lithops/storage/backends/ceph/ceph.py | 4 +- .../backends/gcp_storage/gcp_storage.py | 9 +- lithops/storage/backends/ibm_cos/ibm_cos.py | 6 +- lithops/storage/backends/minio/minio.py | 4 +- .../storage/backends/oracle_oss/oracle_oss.py | 12 +- lithops/storage/backends/swift/swift.py | 290 ++++++++++----- lithops/storage/cloud_proxy.py | 2 +- lithops/tests/test_call_async.py | 27 ++ lithops/tests/test_executors.py | 19 + lithops/tests/test_k8s_backend.py | 22 ++ lithops/tests/test_knative_backend.py | 40 ++ lithops/tests/test_retries.py | 56 +++ lithops/tests/test_serverless.py | 142 ++++++++ lithops/tests/test_standalone.py | 31 ++ lithops/tests/test_storage_layer.py | 342 ++++++++++++++++++ lithops/tests/test_wait.py | 3 + lithops/tests/test_worker.py | 24 ++ lithops/utils.py | 8 + lithops/wait.py | 4 +- lithops/worker/jobrunner.py | 40 +- 33 files changed, 1201 insertions(+), 201 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 86c57739a..78a08dcfb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,12 +12,13 @@ - [Knative] Updated to current Knative Serving (tested with 1.23): one container per call in flight with the current autoscaling annotations, `kubecfg_context`, `namespace` and `runtime_arch` keys, the Contour networking layer, cluster-local calls from inside the cluster, a deploy that updates an existing service and reports why its revision fails, and a default runtime that runs as a non-root user. ### Fixed +- [Core] `get_result()` with no futures returns `[]`. `call_async()` chains a list or tuple of futures as one call. `RetryingFunctionExecutor.wait()` shares one timeout across retry rounds and, when it expires, leaves the calls running instead of cancelling them. - [Core] Chaining, cleanup, invocation errors, generator inputs, empty `fs`, `wait()` with `ANY_COMPLETED`, argument binding, and `RetryingFunctionExecutor` retries and percentage waits; a failed object, or chunk of one, is retried on its own. - [Worker] Partitioned object streams (lost, duplicated or cut rows, rows longer than 128 KB, multi-byte `obj_newline`, `readline()`, iteration), module discovery of nested functions, a parameter named `future`, and `extra_env` restoring the previous environment. - [Localhost] An absolute interpreter path with any character, such as `@` or a space, is accepted as runtime; storage listing is a literal prefix match including dotfiles, and open-ended byte ranges are supported. -- [Storage] `CloudFileProxy` path handling (`isfile`, `listdir`, `exists`, leading `/`), double `close()`, and `r+` mode writing back; Swift `generate_bucket_name`, Aliyun and Oracle listings past 1,000 objects, and open-ended byte ranges on GCS, Azure and Aliyun without modifying the caller's `extra_get_args`; glob patterns outside the bucket root on AWS S3 and IBM COS; Swift listings return `Key` and `Size` like the other backends and go past the first page. -- [Serverless] Oracle Functions reports a missing `oracle_f` section, an Aliyun zip runtime builds without Docker, and Kubernetes no longer waits for ever on a runtime image it cannot pull or a master pod that does not start. -- [Standalone] A failed `init()` on AWS EC2, GCP Compute Engine and IBM VPC now deletes every resource it created, including those whose tagging, attaching or rules failed; IBM VPC no longer uses the deprecated `code` of SDK errors; the `vm` backend loads its documented configuration, and Azure `request_spot_instances` creates Spot worker VMs (now `False` by default: B-series sizes, like the default worker, do not support Spot). +- [Storage] `CloudFileProxy` path handling (`isfile`, `listdir`, `exists`, leading `/`), double `close()`, and `r+` mode writing back; Swift `generate_bucket_name`, Aliyun and Oracle listings past 1,000 objects, and open-ended byte ranges on GCS, Azure and Aliyun without modifying the caller's `extra_get_args`; glob patterns outside the bucket root on AWS S3 and IBM COS; Swift listings return `Key` and `Size` like the other backends and go past the first page. A missing bucket from an S3-compatible listing is `StorageNoSuchKeyError`. Swift uploads and non-404 reads raise, Swift has `head_bucket`, and an empty error page is not the end of a listing. Oracle, Azure and GCP no longer report an arbitrary failure as a missing key or bucket. `CloudFileProxy.exists()` ignores Lithops-internal keys. Swift logs in to current Keystone: a project is scoped by id or by name and domain, the domain defaults to `Default`, and the catalog URL is used as published, including `https`. A container that does not exist is created. +- [Serverless] Oracle Functions reports a missing `oracle_f` section, an Aliyun zip runtime builds without Docker, and Kubernetes no longer waits for ever on a runtime image it cannot pull or a master pod that does not start. The Kubernetes registry secret uses the Docker config keys. A Knative, GCP Functions or Cloud Run deploy that cannot read runtime metadata deletes the service or function it created, and GCP Functions reads that metadata once. An unknown Knative `kubecfg_context` names the contexts that exist. Waiting for a GCP function or Cloud Run service to be deleted treats only HTTP 404 as gone, and a failed Cloud Run delete is reported. +- [Standalone] A failed `init()` on AWS EC2, GCP Compute Engine and IBM VPC now deletes every resource it created, including those whose tagging, attaching or rules failed, and IBM VPC floating IPs allocated by that `init()`; IBM VPC no longer uses the deprecated `code` of SDK errors; the `vm` backend loads its documented configuration, and Azure `request_spot_instances` creates Spot worker VMs (now `False` by default: B-series sizes, like the default worker, do not support Spot). AWS EC2 raises a clear error when spot price history is empty. - [Monitoring] Worker tokens of jobs dropped by `wait()`, replaced monitors and restarted invokers are handed back once, a held `__end__` is no longer replaced by an `__init__`, and the workers of a job that owes no token are forgotten instead of kept for as long as the job monitor runs; the shared RabbitMQ connection of a worker is closed at exit. - [Concurrent] `wait()` and `as_completed()` accept generators. - [Core] An executor that fails to be created no longer logs a misleading `job_monitor` error at exit. diff --git a/config/config_template.yaml b/config/config_template.yaml index 185ac241e..4679bd31e 100644 --- a/config/config_template.yaml +++ b/config/config_template.yaml @@ -532,13 +532,16 @@ # Any additional key in this section is forwarded to redis.Redis(...) #swift: - #auth_url: + #auth_url: # Keystone, with or without a trailing /v3 #region: - #user_id: - #project_id: + #user_name: # Name in user_domain_name. Or set user_id to the user id + #user_id: + #project_name: # Name in project_domain_name. Or set project_id + #project_id: #password: - #user_domain_name: default - #project_domain_name: default + #user_domain_name: Default + #project_domain_name: Default + #endpoint: # Optional. Replaces the URL from the Keystone catalog #storage_bucket: #infinispan: diff --git a/docs/source/storage_config/swift.md b/docs/source/storage_config/swift.md index 236189c90..7b2bdd783 100644 --- a/docs/source/storage_config/swift.md +++ b/docs/source/storage_config/swift.md @@ -21,25 +21,29 @@ Lithops with OpenStack Swift as storage backend. swift: storage_bucket: - auth_url : - region : - user_id : - project_id : - password : - user_domain_name: - project_domain_name: + auth_url: + region: + user_name: + project_name: + password: + user_domain_name: Default + project_domain_name: Default ``` - + +`user_id` and `project_id` identify the user and the project by id instead of by name. Set one of each pair. `auth_url` may or may not end in `/v3`. `endpoint` replaces the storage URL published in the Keystone catalog. ## Summary of configuration keys for Swift: |Group|Key|Default|Mandatory|Additional info| |---|---|---|---|---| |swift | storage_bucket | | yes | The name of a container that exists in your account. This will be used by Lithops for intermediate data. If set, this will overwrite the `storage_bucket` set in the `lithops` section | -|swift | auth_url | |yes | The keystone endpoint for authentication | -|swift | region | |yes | The region of your container | -|swift | project_id | |yes | The Project ID | -|swift | user_id | |yes | The user ID | -|swift | password | |yes | The password | -|swift | user_domain_name | | no | The domain to which the user belongs, by default is set to "default" | -|swift | project_domain_name | | no | The domain associated with the project, by default is set to "default" | +|swift | auth_url | | yes | The Keystone endpoint. A trailing `/v3` is optional | +|swift | region | | yes | The region of the public Swift endpoint in the catalog | +|swift | user_name | | no | The user name. Required unless `user_id` is set | +|swift | user_id | | no | The user id. A value that is not an id is used as the user name | +|swift | project_name | | no | The project name. Required unless `project_id` is set | +|swift | project_id | | no | The project id. A value that is not an id is used as the project name | +|swift | password | | yes | The password | +|swift | user_domain_name | Default | no | The domain of a user named with `user_name` | +|swift | project_domain_name | Default | no | The domain of a project named with `project_name` | +|swift | endpoint | | no | Storage URL. Replaces the public URL from the Keystone catalog | diff --git a/lithops/executors.py b/lithops/executors.py index 978d80886..4b25102c1 100644 --- a/lithops/executors.py +++ b/lithops/executors.py @@ -55,6 +55,7 @@ create_futures_list, FuturesList, _as_future_list, + _CALL_ASYNC_CHAIN, _chained_futures, log_prefix, ) @@ -481,6 +482,18 @@ def _submit_map( self._disable_iterdata_output(iterdata, futures) return job_id, job, futures + def _cancel_wait(self, futures, do_clean, exception): + """ + Gives up on the futures a wait was blocked on: they are dropped from + the monitor, marked unknown and, unless asked not to, cleaned + """ + self.job_monitor.remove(futures) + for future in futures: + future._set_exception() + self._stop_monitor_if_idle(futures) + if do_clean: + self._cleanup_jobs(futures, exception=exception, force=True) + def _cleanup_jobs(self, futures, exception=None, force=False): """ Releases the backend resources of the given jobs and deletes their @@ -542,9 +555,15 @@ def call_async( :return: Response future. """ self.last_call = 'call_async' + # A list or tuple of futures is one chained call: map() would make + # one activation per future, and this returns a single future + if isinstance(data, (list, tuple)) and _chained_futures(data) is not None: + submitted = [{_CALL_ASYNC_CHAIN: data}] + else: + submitted = [data] _, _, futures = self._submit_map( func, - [data], + submitted, runtime_memory=runtime_memory, extra_env=extra_env, include_modules=include_modules, @@ -760,7 +779,8 @@ def wait( threadpool_size: Optional[int] = THREADPOOL_SIZE, wait_dur_sec: Optional[int] = None, show_progressbar: Optional[bool] = True, - clean_jobs: Optional[bool] = None + clean_jobs: Optional[bool] = None, + cancel_on_timeout: bool = True, ) -> Tuple[FuturesList, FuturesList]: """ Wait for the Future instances (possibly created by @@ -791,6 +811,10 @@ def wait( future is done. Default is ``data_cleaner``. ``get_result()`` passes False so it can log that results are in before it cleans. + :param cancel_on_timeout: On timeout, drop the waited jobs and + raise. False returns the calls that finished and leaves the + rest running, which is what a retrying wait needs so it can + resubmit the failures inside the time it has left. :return: `(fs_done, fs_notdone)` where `fs_done` is a list of futures that have completed and `fs_notdone` @@ -821,19 +845,30 @@ def wait( if do_clean and return_when == ALL_COMPLETED: self._cleanup_jobs(futures) - except (KeyboardInterrupt, Exception) as e: - if isinstance(e, KeyboardInterrupt): - self.invoker.stop(wait=True) - else: - # Only the jobs waited on end here. Another job of this - # executor may still have calls queued for a free worker - self.invoker.discard_pending({f.job_key for f in futures}) - self.job_monitor.remove(futures) - for future in futures: - future._set_exception() - self._stop_monitor_if_idle(futures) - if do_clean: - self._cleanup_jobs(futures, exception=e, force=True) + except TimeoutError as e: + if not cancel_on_timeout: + # The calls keep running, but ones that already reported + # back must leave the monitor. A retry replaces them, and + # the next wait would never be asked to release the old ones + self._release_finished_from_monitor(futures) + fs_done, fs_notdone = _partition_futures( + futures, download_results + ) + return ( + create_futures_list(fs_done, self), + create_futures_list(fs_notdone, self), + ) + self._cancel_wait(futures, do_clean, e) + raise + except KeyboardInterrupt as e: + self.invoker.stop(wait=True) + self._cancel_wait(futures, do_clean, e) + raise + except Exception as e: + # Only the jobs waited on end here. Another job of this + # executor may still have calls queued for a free worker + self.invoker.discard_pending({f.job_key for f in futures}) + self._cancel_wait(futures, do_clean, e) raise fs_done, fs_notdone = _partition_futures(futures, download_results) diff --git a/lithops/retries.py b/lithops/retries.py index fe641231c..fe064afea 100644 --- a/lithops/retries.py +++ b/lithops/retries.py @@ -12,6 +12,7 @@ # limitations under the License. # +import time from collections.abc import Iterator from typing import Any, Callable, Dict, List, Optional, Tuple, Union @@ -340,6 +341,17 @@ def _split_done_and_retried(self, done, pending, lookup): return retrying_done, retrying_pending + @staticmethod + def _split_without_retry(done, pending, lookup): + """ + The same split as a finished round, without resubmitting failures. + Used once the deadline has passed: a retry would run past the timeout + """ + return ( + [lookup[response_future] for response_future in done], + [lookup[response_future] for response_future in pending], + ) + @staticmethod def _wait_is_over(return_when, retrying_done, retrying_pending): """ @@ -387,26 +399,39 @@ def wait( if not fs: return [], [] lookup = {f.response_future: f for f in fs} + # One deadline for every retry round. Passing the original timeout + # again gave each round a fresh budget + deadline = None if timeout is None else time.monotonic() + timeout while True: + remaining = None if deadline is None else max(0, deadline - time.monotonic()) # A retry replaces the response future of a RetryingFuture, so - # the list has to be rebuilt on every round. The inner wait never - # raises: a call that failed would end the whole wait there, and - # the executor would discard every future instead of retrying it + # the list has to be rebuilt on every round. A call that failed + # must come back here instead of ending the wait: the executor + # would otherwise discard every future instead of retrying it. + # A timeout returns what finished and leaves the rest running, + # so this loop can still retry inside the time that is left done, pending = self.executor.wait( [f.response_future for f in fs], throw_except=False, return_when=return_when, download_results=download_results, - timeout=timeout, + timeout=remaining, threadpool_size=threadpool_size, wait_dur_sec=wait_dur_sec, show_progressbar=show_progressbar, + cancel_on_timeout=False, ) - retrying_done, retrying_pending = self._split_done_and_retried( - done, pending, lookup - ) + out_of_time = deadline is not None and time.monotonic() >= deadline + if out_of_time: + retrying_done, retrying_pending = self._split_without_retry( + done, pending, lookup + ) + else: + retrying_done, retrying_pending = self._split_done_and_retried( + done, pending, lookup + ) if self._wait_is_over( return_when, retrying_done, retrying_pending @@ -417,6 +442,12 @@ def wait( retrying_future._reraise_if_error() return retrying_done, retrying_pending + if out_of_time: + raise TimeoutError( + f'Timeout of {timeout} seconds exceeded waiting for ' + 'function activations to finish' + ) + def clean( self, fs: Optional[Union[ResponseFuture, List[ResponseFuture]]] = None, diff --git a/lithops/serverless/backends/gcp_cloudrun/cloudrun.py b/lithops/serverless/backends/gcp_cloudrun/cloudrun.py index 701894357..8f5510e5b 100644 --- a/lithops/serverless/backends/gcp_cloudrun/cloudrun.py +++ b/lithops/serverless/backends/gcp_cloudrun/cloudrun.py @@ -457,7 +457,19 @@ def deploy_runtime(self, runtime_name, memory, timeout): img_name = self._format_image_name(runtime_name) logger.info(f"Deploying runtime: {img_name} - Memory: {memory} Timeout: {timeout}") self._create_service(runtime_name, memory, timeout) - runtime_meta = self._generate_runtime_meta(runtime_name, memory) + try: + runtime_meta = self._generate_runtime_meta(runtime_name, memory) + except Exception: + # The service is already live. Leaving it behind makes the next + # deploy look like an update of a runtime that has no metadata + logger.warning(f'Deleting {runtime_name}: its metadata could not be read') + try: + self.delete_runtime(runtime_name, memory) + except Exception as cleanup_error: + logger.warning( + f'Could not delete the Cloud Run service for {runtime_name}: {cleanup_error}' + ) + raise return runtime_meta def delete_runtime(self, runtime_name, runtime_memory, version=__version__): @@ -468,22 +480,26 @@ def delete_runtime(self, runtime_name, runtime_memory, version=__version__): def _delete_service(self, service_name): logger.debug(f'Deleting service {service_name}') + name = f'namespaces/{self.project_name}/services/{service_name}' try: - self._api_resource.namespaces().services().delete( - name=f'namespaces/{self.project_name}/services/{service_name}' - ).execute() - # Wait until the service is completely deleted - while True: - try: - self._api_resource.namespaces().services().get( - name=f'namespaces/{self.project_name}/services/{service_name}' - ).execute() - time.sleep(1) - except Exception: - break - logger.debug(f'Ok -- service deleted {service_name}') - except Exception: - logger.debug(f'Error -- unable to delete service {service_name}') + self._api_resource.namespaces().services().delete(name=name).execute() + except HttpError as e: + if getattr(getattr(e, 'resp', None), 'status', None) == 404: + logger.debug(f'Service {service_name} is already gone') + return + raise + + # Wait until the service is completely deleted. Anything other than + # 404 (a blip, a permission error) is not evidence that it is gone + while True: + try: + self._api_resource.namespaces().services().get(name=name).execute() + time.sleep(1) + except HttpError as e: + if getattr(getattr(e, 'resp', None), 'status', None) == 404: + logger.debug(f'Ok -- service deleted {service_name}') + return + raise def clean(self, **kwargs): logger.debug('Going to delete all deployed runtimes') diff --git a/lithops/serverless/backends/gcp_functions/gcp_functions.py b/lithops/serverless/backends/gcp_functions/gcp_functions.py index a4fc462d4..6da570d39 100644 --- a/lithops/serverless/backends/gcp_functions/gcp_functions.py +++ b/lithops/serverless/backends/gcp_functions/gcp_functions.py @@ -187,16 +187,20 @@ def _list_built_runtimes(self, default_runtimes=True): return runtimes def _wait_function_deleted(self, function_location): - # Wait until function is completely deleted + # Wait until function is completely deleted. Anything other than + # 404 (a blip, a permission error) is not evidence that it is gone while True: try: self._api_resource.projects().locations().functions().get( name=function_location ).execute(num_retries=self.num_retries) time.sleep(self.retry_sleep) - except Exception: - logger.debug('Function status is DELETED') - break + except HttpError as e: + status = getattr(getattr(e, 'resp', None), 'status', None) + if status == 404: + logger.debug('Function status is DELETED') + return + raise def _create_function(self, runtime_name, memory, timeout=60): """ @@ -351,9 +355,19 @@ def deploy_runtime(self, runtime_name, memory, timeout): os.remove(runtime_path) self._create_function(runtime_name, memory, timeout) - - # Get runtime metadata - runtime_meta = self._generate_runtime_meta(runtime_name, memory) + try: + runtime_meta = self._generate_runtime_meta(runtime_name, memory) + except Exception: + # The function is already live. Leaving it behind makes the next + # deploy look like an update of a runtime that has no metadata + logger.warning(f'Deleting {runtime_name}: its metadata could not be read') + try: + self.delete_runtime(runtime_name, memory) + except Exception as cleanup_error: + logger.warning( + f'Could not delete the function for {runtime_name}: {cleanup_error}' + ) + raise return runtime_meta @@ -515,9 +529,6 @@ def _generate_runtime_meta(self, runtime_name, runtime_memory): raise Exception(f'Failed getting runtime metadata: {runtime_meta}') logger.debug(f'Ok -- Extraced modules from {runtime_name}') - - runtime_meta = self.invoke(runtime_name, runtime_memory, payload, True) - return runtime_meta def get_runtime_key(self, runtime_name, runtime_memory, version=__version__): diff --git a/lithops/serverless/backends/k8s/k8s.py b/lithops/serverless/backends/k8s/k8s.py index d2f0c36c7..f7746b646 100644 --- a/lithops/serverless/backends/k8s/k8s.py +++ b/lithops/serverless/backends/k8s/k8s.py @@ -292,8 +292,11 @@ def _create_container_registry_secret(self): cred_payload = { "auths": { docker_server: { - "Username": docker_user, - "Password": docker_password + "username": docker_user, + "password": docker_password, + "auth": base64.b64encode( + f'{docker_user}:{docker_password}'.encode() + ).decode() } } } diff --git a/lithops/serverless/backends/knative/knative.py b/lithops/serverless/backends/knative/knative.py index 0bde3f1c6..e5d653c77 100644 --- a/lithops/serverless/backends/knative/knative.py +++ b/lithops/serverless/backends/knative/knative.py @@ -119,7 +119,14 @@ def __init__(self, knative_config, internal_storage): load_kube_config(config_file=self.kubecfg_path, context=context) contexts, current_context = list_kube_config_contexts(config_file=self.kubecfg_path) if context is not None: - current_context = [it for it in contexts if it['name'] == context][0] + matches = [it for it in contexts if it['name'] == context] + if not matches: + known = ', '.join(it['name'] for it in contexts) or '(none)' + raise Exception( + f'Kubernetes context {context!r} was not found in ' + f'{self.kubecfg_path}. Known contexts: {known}' + ) + current_context = matches[0] ctx_context = current_context.get('context') self.namespace = self.namespace or ctx_context.get('namespace') self.cluster = ctx_context.get('cluster') or self.cluster @@ -529,7 +536,19 @@ def deploy_runtime(self, runtime_name, memory, timeout): logger.info(f"Deploying runtime: {runtime_name} - Memory: {memory} Timeout: {timeout}") self._create_container_registry_secret() self._create_service(runtime_name, memory, timeout) - runtime_meta = self._generate_runtime_meta(runtime_name, memory) + try: + runtime_meta = self._generate_runtime_meta(runtime_name, memory) + except Exception: + # The service is already live. Leaving it behind makes the next + # deploy look like an update of a runtime that has no metadata + logger.warning(f'Deleting {runtime_name}: its metadata could not be read') + try: + self.delete_runtime(runtime_name, memory) + except Exception as cleanup_error: + logger.warning( + f'Could not delete the Knative service for {runtime_name}: {cleanup_error}' + ) + raise return runtime_meta diff --git a/lithops/standalone/backends/aws_ec2/aws_ec2.py b/lithops/standalone/backends/aws_ec2/aws_ec2.py index 24fc49201..ded36cd4d 100644 --- a/lithops/standalone/backends/aws_ec2/aws_ec2.py +++ b/lithops/standalone/backends/aws_ec2/aws_ec2.py @@ -504,9 +504,15 @@ def _request_spot_price(self): ProductDescriptions=['Linux/UNIX (Amazon VPC)'], StartTime=datetime.today() ) - spot_prices = [] - for az in response['SpotPriceHistory']: - spot_prices.append(float(az['SpotPrice'])) + spot_prices = [ + float(az['SpotPrice']) for az in response['SpotPriceHistory'] + ] + if not spot_prices: + raise ValueError( + f'No spot price history for worker instance type {wit}. ' + 'Check the instance type and region, or set ' + 'request_spot_instances to false' + ) self.config["spot_price"] = max(spot_prices) logger.debug(f'Current spot instance price for {wit} is ${self.config["spot_price"]}') diff --git a/lithops/standalone/backends/ibm_vpc/ibm_vpc.py b/lithops/standalone/backends/ibm_vpc/ibm_vpc.py index d54ec306b..58070e3b1 100644 --- a/lithops/standalone/backends/ibm_vpc/ibm_vpc.py +++ b/lithops/standalone/backends/ibm_vpc/ibm_vpc.py @@ -186,6 +186,12 @@ def _rollback_init_resources(self): f'VPC {vpc_name}', ) + for fip_id in list(getattr(self, '_init_created_fips', ())): + self._safe_rollback_delete( + lambda fid=fip_id: self.vpc_cli.delete_floating_ip(id=fid), + f'floating IP {fip_id}', + ) + if 'ssh_key' in created: key_filename = self.config.get('ssh_key_filename') if key_filename and 'lithops-key-' in key_filename: @@ -454,6 +460,11 @@ def _get_or_create_floating_ip(self): floating_ip_prototype['resource_group'] = {'id': self.config['resource_group_id']} response = self.vpc_cli.create_floating_ip(floating_ip_prototype) fip_data = response.result + # Only an address this init() allocated is deleted if init fails. + # A recyclable one that was already free is left for the next use + created = getattr(self, '_init_created_fips', None) + if created is not None: + created.add(fip_data['id']) return fip_data['address'], fip_data['id'] @@ -561,6 +572,7 @@ def init(self): # The resources this init() creates, deleted again if it fails self._init_created = set() + self._init_created_fips = set() try: # Create the VPC if not exists self._create_vpc() @@ -606,6 +618,7 @@ def init(self): raise finally: self._init_created = None + self._init_created_fips = None def build_image(self, image_name, script_file, overwrite, include, extra_args=[]): """ diff --git a/lithops/storage/backends/aws_s3/aws_s3.py b/lithops/storage/backends/aws_s3/aws_s3.py index 58b6eedf5..1b285ea33 100644 --- a/lithops/storage/backends/aws_s3/aws_s3.py +++ b/lithops/storage/backends/aws_s3/aws_s3.py @@ -246,7 +246,9 @@ def list_objects(self, bucket_name, prefix=None, match_pattern=None): object_list.extend(page['Contents']) return object_list except botocore.exceptions.ClientError as e: - if e.response['Error']['Code'] == '404': + # HeadObject reports a missing key as '404'. ListObjectsV2 + # reports a missing bucket as 'NoSuchBucket' + if e.response['Error']['Code'] in ('404', 'NoSuchBucket'): raise StorageNoSuchKeyError( bucket_name, '' if prefix is None else prefix) else: @@ -272,7 +274,7 @@ def list_keys(self, bucket_name, prefix=None): key_list.append(item['Key']) return key_list except botocore.exceptions.ClientError as e: - if e.response['Error']['Code'] == '404': + if e.response['Error']['Code'] in ('404', 'NoSuchBucket'): raise StorageNoSuchKeyError(bucket_name, prefix) else: raise e diff --git a/lithops/storage/backends/azure_storage/azure_storage.py b/lithops/storage/backends/azure_storage/azure_storage.py index 1bd1d6688..6146e4fec 100644 --- a/lithops/storage/backends/azure_storage/azure_storage.py +++ b/lithops/storage/backends/azure_storage/azure_storage.py @@ -173,9 +173,10 @@ def head_object(self, bucket_name, key): container_client = self.blob_client.get_container_client(bucket_name) blob = container_client.get_blob_client(key) - # adapted to match ibm_cos method - metadata = {} - metadata['content-length'] = blob.get_blob_properties().size + try: + metadata = {'content-length': blob.get_blob_properties().size} + except ResourceNotFoundError: + raise StorageNoSuchKeyError(bucket_name, key) return metadata def delete_object(self, bucket_name, key): @@ -213,7 +214,7 @@ def head_bucket(self, bucket_name): try: container_client = self.blob_client.get_container_client(bucket_name) return container_client.get_container_properties() - except Exception: + except ResourceNotFoundError: raise StorageNoSuchKeyError(bucket_name, '') def list_objects(self, bucket_name, prefix=None, match_pattern=None): @@ -235,7 +236,7 @@ def list_objects(self, bucket_name, prefix=None, match_pattern=None): 'Size': blob.size }) return mod_list - except Exception: + except ResourceNotFoundError: raise StorageNoSuchKeyError(bucket_name, '' if prefix is None else prefix) def list_keys(self, bucket_name, prefix=None): @@ -249,5 +250,5 @@ def list_keys(self, bucket_name, prefix=None): container_client = self.blob_client.get_container_client(bucket_name) keys = [blob.name for blob in container_client.list_blobs(prefix)] return keys - except Exception: + except ResourceNotFoundError: raise StorageNoSuchKeyError(bucket_name, '' if prefix is None else prefix) diff --git a/lithops/storage/backends/ceph/ceph.py b/lithops/storage/backends/ceph/ceph.py index 7aff2cb2e..37c8205b5 100644 --- a/lithops/storage/backends/ceph/ceph.py +++ b/lithops/storage/backends/ceph/ceph.py @@ -270,7 +270,7 @@ def list_objects(self, bucket_name, prefix=None, match_pattern=None): object_list.append(item) return object_list except botocore.exceptions.ClientError as e: - if e.response['Error']['Code'] == '404': + if e.response['Error']['Code'] in ('404', 'NoSuchBucket'): raise StorageNoSuchKeyError(bucket_name, '' if prefix is None else prefix) else: raise e @@ -295,7 +295,7 @@ def list_keys(self, bucket_name, prefix=None): key_list.append(item['Key']) return key_list except botocore.exceptions.ClientError as e: - if e.response['Error']['Code'] == '404': + if e.response['Error']['Code'] in ('404', 'NoSuchBucket'): raise StorageNoSuchKeyError(bucket_name, prefix) else: raise e diff --git a/lithops/storage/backends/gcp_storage/gcp_storage.py b/lithops/storage/backends/gcp_storage/gcp_storage.py index ec9bc4b5b..7cc8d6a81 100644 --- a/lithops/storage/backends/gcp_storage/gcp_storage.py +++ b/lithops/storage/backends/gcp_storage/gcp_storage.py @@ -209,7 +209,10 @@ def delete_objects(self, bucket_name, key_list): pass def head_bucket(self, bucket_name): - bucket = self.client.get_bucket(bucket_name, timeout=TIMEOUT) + try: + bucket = self.client.get_bucket(bucket_name, timeout=TIMEOUT) + except google_exceptions.NotFound: + raise StorageNoSuchKeyError(bucket_name, '') response = { 'ResponseMetadata': {'HTTPStatusCode': 200, @@ -223,7 +226,7 @@ def list_objects(self, bucket_name, prefix=None, match_pattern=None): try: bucket = self.client.get_bucket(bucket_name, timeout=TIMEOUT) page = bucket.list_blobs(prefix=prefix) - except google_exceptions.ClientError: + except google_exceptions.NotFound: raise StorageNoSuchKeyError(bucket_name, '') return [{'Key': blob.name, 'Size': blob.size, 'LastModified': blob.updated} for blob in page] @@ -231,6 +234,6 @@ def list_keys(self, bucket_name, prefix=None): try: bucket = self.client.get_bucket(bucket_name, timeout=TIMEOUT) page = bucket.list_blobs(prefix=prefix) - except google_exceptions.ClientError: + except google_exceptions.NotFound: raise StorageNoSuchKeyError(bucket_name, '') return [blob.name for blob in page] diff --git a/lithops/storage/backends/ibm_cos/ibm_cos.py b/lithops/storage/backends/ibm_cos/ibm_cos.py index 3174c462c..ecc98da6f 100644 --- a/lithops/storage/backends/ibm_cos/ibm_cos.py +++ b/lithops/storage/backends/ibm_cos/ibm_cos.py @@ -318,7 +318,9 @@ def list_objects(self, bucket_name, prefix=None, match_pattern=None): object_list.extend(page['Contents']) return object_list except ibm_botocore.exceptions.ClientError as e: - if e.response['Error']['Code'] == '404': + # HeadObject reports a missing key as '404'. ListObjectsV2 + # reports a missing bucket as 'NoSuchBucket' + if e.response['Error']['Code'] in ('404', 'NoSuchBucket'): raise StorageNoSuchKeyError(bucket_name, '' if prefix is None else prefix) else: raise e @@ -343,7 +345,7 @@ def list_keys(self, bucket_name, prefix=None): key_list.append(item['Key']) return key_list except ibm_botocore.exceptions.ClientError as e: - if e.response['Error']['Code'] == '404': + if e.response['Error']['Code'] in ('404', 'NoSuchBucket'): raise StorageNoSuchKeyError(bucket_name, prefix) else: raise e diff --git a/lithops/storage/backends/minio/minio.py b/lithops/storage/backends/minio/minio.py index 53654d0da..5525734a2 100644 --- a/lithops/storage/backends/minio/minio.py +++ b/lithops/storage/backends/minio/minio.py @@ -268,7 +268,7 @@ def list_objects(self, bucket_name, prefix=None, match_pattern=None): object_list.append(item) return object_list except botocore.exceptions.ClientError as e: - if e.response['Error']['Code'] == '404': + if e.response['Error']['Code'] in ('404', 'NoSuchBucket'): raise StorageNoSuchKeyError(bucket_name, '' if prefix is None else prefix) else: raise e @@ -293,7 +293,7 @@ def list_keys(self, bucket_name, prefix=None): key_list.append(item['Key']) return key_list except botocore.exceptions.ClientError as e: - if e.response['Error']['Code'] == '404': + if e.response['Error']['Code'] in ('404', 'NoSuchBucket'): raise StorageNoSuchKeyError(bucket_name, prefix) else: raise e diff --git a/lithops/storage/backends/oracle_oss/oracle_oss.py b/lithops/storage/backends/oracle_oss/oracle_oss.py index d35d881b3..47791cdc8 100644 --- a/lithops/storage/backends/oracle_oss/oracle_oss.py +++ b/lithops/storage/backends/oracle_oss/oracle_oss.py @@ -104,8 +104,9 @@ def put_object(self, bucket_name, key, data): logger.debug('OSS Object {} uploaded to bucket {} - Size: {}'.format( key, bucket_name, sizeof_fmt(len(data)))) except oci.exceptions.ServiceError as e: - logger.debug("ServiceError in put_object: %s", str(e)) - raise StorageNoSuchKeyError(bucket_name, key) + if e.status == 404: + raise StorageNoSuchKeyError(bucket_name, key) + raise def get_object(self, bucket_name, key, stream=False, extra_get_args={}): ''' @@ -255,7 +256,7 @@ def list_objects(self, bucket_name, prefix=None, match_pattern=None): :return: A list of dictionaries containing the keys and sizes of the objects that match the given prefix and match pattern :rtype: list of dict - :raises StorageNoSuchKeyError: If the specified bucket does not exist or there is a service error + :raises StorageNoSuchKeyError: If the specified bucket does not exist ''' prefix = '' if prefix is None else prefix try: @@ -272,8 +273,9 @@ def list_objects(self, bucket_name, prefix=None, match_pattern=None): next_start = res.data.next_start_with except oci.exceptions.ServiceError as e: - logger.debug("ServiceError in list_objects: %s", str(e)) - raise StorageNoSuchKeyError(bucket_name, prefix) + if e.status == 404: + raise StorageNoSuchKeyError(bucket_name, prefix) + raise def list_keys(self, bucket_name, prefix=None): ''' diff --git a/lithops/storage/backends/swift/swift.py b/lithops/storage/backends/swift/swift.py index 4f6e47829..3490b7e3e 100644 --- a/lithops/storage/backends/swift/swift.py +++ b/lithops/storage/backends/swift/swift.py @@ -15,16 +15,24 @@ # import os +import re import json import shutil import logging import requests +from urllib.parse import urlsplit, urlunsplit + from lithops.storage.utils import StorageNoSuchKeyError from lithops.utils import sizeof_fmt from lithops.constants import STORAGE_CLI_MSG logger = logging.getLogger(__name__) +# Keystone ids are 32 hex characters, or a UUID with hyphens +_OPENSTACK_ID = re.compile( + r'^(?:[0-9a-fA-F]{32}|[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12})$' +) + class StorageBackend: """ @@ -34,20 +42,25 @@ class StorageBackend: def __init__(self, swift_config): logger.debug("Creating OpenStack Swift client") self.config = swift_config - self.auth_url = swift_config['auth_url'] - self.user_id = swift_config['user_id'] - self.project_id = swift_config['project_id'] - self.password = swift_config['password'] - self.region = swift_config['region'] - self.user_domain_name = swift_config.get("user_domain_name", "default") - self.project_domain_name = swift_config.get("project_domain_name", "default") - self.endpoint = None - - if 'token' in swift_config: + self.auth_url = swift_config['auth_url'].rstrip('/') + self.user_id = swift_config.get('user_id') + self.project_id = swift_config.get('project_id') + self.password = swift_config.get('password') + self.region = swift_config.get('region') + # Keystone's own default domain is named Default. "default" does not match it + self.user_domain_name = swift_config.get('user_domain_name', 'Default') + self.project_domain_name = swift_config.get('project_domain_name', 'Default') + configured_endpoint = swift_config.get('endpoint') + + if swift_config.get('token'): + if not configured_endpoint: + raise Exception('A Swift token also needs the storage endpoint') self.token = swift_config['token'] - self.endpoint = swift_config['endpoint'] + self.endpoint = configured_endpoint else: self.token = self.generate_swift_token() + if configured_endpoint: + self.endpoint = configured_endpoint swift_config['token'] = self.token swift_config['endpoint'] = self.endpoint @@ -64,58 +77,140 @@ def generate_bucket_name(self): """ Generates a unique bucket name """ - self.config['storage_bucket'] = f'lithops-{self.region}-{self.user_id[:6].lower()}' + owner = self.config.get('user_name') or self.user_id or 'user' + self.config['storage_bucket'] = f'lithops-{self.region}-{owner[:6].lower()}' return self.config['storage_bucket'] - def generate_swift_token(self): + def get_client(self): + """The requests session that carries the Keystone token""" + return self.session + + def create_bucket(self, bucket_name): """ - Generates new token for accessing to Swift. - :return: token + Create a container. Swift answers 202 when it already exists """ - url = self.auth_url + "/v3/auth/tokens" - headers = {'Content-Type': 'application/json'} - data = { - "auth": { - "identity": { - "methods": ["password"], - "password": { - "user": { - "name": self.user_id, - "password": self.password, - "domain": {"name": self.user_domain_name} - } - } - }, - "scope": { - "project": { - "id": self.project_id, - "domain": {"name": self.project_domain_name} - } - } + url = '/'.join([self.endpoint.rstrip('/'), bucket_name]) + res = self.session.put(url) + if res.status_code not in (201, 202): + raise Exception( + f'PUT container {bucket_name} failed with status {res.status_code}' + ) + + def _token_url(self): + """Keystone identity URL. auth_url may already end in /v3""" + if self.auth_url.endswith('/v3'): + return self.auth_url + '/auth/tokens' + return self.auth_url + '/v3/auth/tokens' + + def _user_identity(self): + """ + A user is an id, or a name in a domain. Keystone rejects a domain + on a user that was named by id + """ + user_name = self.config.get('user_name') + if user_name: + return { + 'name': user_name, + 'password': self.password, + 'domain': {'name': self.user_domain_name}, } + if _OPENSTACK_ID.match(self.user_id or ''): + return {'id': self.user_id, 'password': self.password} + if not self.user_id: + raise Exception('Swift config needs user_id or user_name') + # user_id has historically held the user name + return { + 'name': self.user_id, + 'password': self.password, + 'domain': {'name': self.user_domain_name}, } - json_data = json.dumps(data) - r = requests.post(url, data=json_data, headers=headers) + def _project_scope(self): + """A project is an id, or a name in a domain. The two cannot be mixed""" + project_name = self.config.get('project_name') + if project_name: + return { + 'name': project_name, + 'domain': {'name': self.project_domain_name}, + } + if _OPENSTACK_ID.match(self.project_id or ''): + return {'id': self.project_id} + if not self.project_id: + raise Exception('Swift config needs project_id or project_name') + return { + 'name': self.project_id, + 'domain': {'name': self.project_domain_name}, + } - if r.status_code == 201: - backend_info = json.loads(r.text) + def _select_endpoint(self, catalog, project_id): + """The public object-store URL for the configured region""" + known = [] + for service in catalog: + if service.get('type') != 'object-store' and service.get('name') != 'swift': + continue + for endpoint in service.get('endpoints', []): + region = endpoint.get('region_id') or endpoint.get('region') + known.append(region or '') + if endpoint.get('interface') != 'public': + continue + if self.region and region != self.region: + continue + url = endpoint['url'] + if project_id: + url = url.replace('%(project_id)s', project_id) + url = url.replace('%(tenant_id)s', project_id) + return self._endpoint_reachable_from_here(url) + regions = ', '.join(sorted(set(known))) or 'none' + raise Exception( + f'No public Swift endpoint for region {self.region!r}. ' + f'Known regions: {regions}' + ) - for service in backend_info['token']['catalog']: - if service['name'] == 'swift': - for endpoint in service['endpoints']: - if endpoint['region'] == self.region: - if endpoint['interface'] == 'public': - self.endpoint = endpoint['url'].replace('https:', 'http:') + def _endpoint_reachable_from_here(self, url): + """ + A catalog entry of 0.0.0.0 is the address the server bound, not an + address a client can open. Use the host Keystone itself was reached on + """ + parts = urlsplit(url) + if parts.hostname != '0.0.0.0': + return url + auth_host = urlsplit(self.auth_url).hostname + if not auth_host: + return url + netloc = auth_host if parts.port is None else f'{auth_host}:{parts.port}' + return urlunsplit((parts.scheme, netloc, parts.path, parts.query, parts.fragment)) - if not self.endpoint: - raise Exception('Invalid region name') + def generate_swift_token(self): + """ + Generates new token for accessing to Swift. + :return: token + """ + payload = { + 'auth': { + 'identity': { + 'methods': ['password'], + 'password': {'user': self._user_identity()}, + }, + 'scope': {'project': self._project_scope()}, + } + } + response = requests.post( + self._token_url(), + data=json.dumps(payload), + headers={'Content-Type': 'application/json'}, + ) + if response.status_code != 201: + try: + message = response.json()['error']['message'] + except Exception: + message = response.text + raise Exception(f'{response.status_code} - {message}') - return r.headers['X-Subject-Token'] - else: - message = json.loads(r.text)['error']['message'] - raise Exception("{} - {} - {}".format(r.status_code, r.reason, message)) + body = response.json() + project_id = (body.get('token') or {}).get('project', {}).get('id') + self.endpoint = self._select_endpoint(body['token']['catalog'], project_id) + return response.headers['X-Subject-Token'] def put_object(self, bucket_name, key, data): """ @@ -126,15 +221,13 @@ def put_object(self, bucket_name, key, data): :return: None """ url = '/'.join([self.endpoint, bucket_name, key]) + res = self.session.put(url, data=data) + if res.status_code != 201: + raise Exception(f'PUT {key} failed with status {res.status_code}') try: - res = self.session.put(url, data=data) - status = 'OK' if res.status_code == 201 else 'Error' - try: - logger.debug('PUT Object {} - Size: {} - {}'.format(key, sizeof_fmt(len(data)), status)) - except Exception: - logger.debug('PUT Object {} - {}'.format(key, status)) - except Exception as e: - print(e) + logger.debug('PUT Object {} - Size: {} - {}'.format(key, sizeof_fmt(len(data)), 'OK')) + except Exception: + logger.debug('PUT Object {} - OK'.format(key)) def get_object(self, bucket_name, key, stream=False, extra_get_args={}): """ @@ -146,23 +239,12 @@ def get_object(self, bucket_name, key, stream=False, extra_get_args={}): url = '/'.join([self.endpoint, bucket_name, key]) headers = {'X-Auth-Token': self.token} headers.update(extra_get_args) - try: - res = self.session.get(url, headers=headers, stream=stream) - if res.status_code == 200 or res.status_code == 206: - if stream: - data = res.raw - else: - data = res.content - return data - elif res.status_code == 404: - raise StorageNoSuchKeyError(bucket_name, key) - else: - raise Exception('{} - {}'.format(res.status_code, key)) - except StorageNoSuchKeyError: - raise - except Exception as e: - print(e) + res = self.session.get(url, headers=headers, stream=stream) + if res.status_code in (200, 206): + return res.raw if stream else res.content + if res.status_code == 404: raise StorageNoSuchKeyError(bucket_name, key) + raise Exception('{} - {}'.format(res.status_code, key)) def upload_file(self, file_name, bucket, key=None, extra_args={}, config=None): """Upload a file @@ -218,16 +300,12 @@ def head_object(self, bucket_name, key): :rtype: str/bytes """ url = '/'.join([self.endpoint, bucket_name, key]) - try: - res = self.session.head(url) - if res.status_code == 200: - return res.headers - elif res.status_code == 404: - raise StorageNoSuchKeyError(bucket_name, key) - else: - raise Exception('{} - {}'.format(res.status_code, key)) - except Exception: + res = self.session.head(url) + if res.status_code == 200: + return res.headers + if res.status_code == 404: raise StorageNoSuchKeyError(bucket_name, key) + raise Exception('{} - {}'.format(res.status_code, key)) def delete_object(self, bucket_name, key): """ @@ -236,7 +314,10 @@ def delete_object(self, bucket_name, key): :param key: data key """ url = '/'.join([self.endpoint, bucket_name, key]) - return self.session.delete(url) + res = self.session.delete(url) + if res.status_code not in (204, 404): + raise Exception(f'{res.status_code} - {key}') + return res def delete_objects(self, bucket_name, key_list): """ @@ -252,12 +333,31 @@ def delete_objects(self, bucket_name, key_list): keys_to_delete.append('/{}/{}'.format(bucket_name, key)) keys_to_delete = '\n'.join(keys_to_delete) - url = '/'.join([self.endpoint, '?bulk-delete']) - return self.session.delete(url, data=keys_to_delete, headers=headers) + url = self.endpoint.rstrip('/') + '/?bulk-delete' + res = self.session.delete(url, data=keys_to_delete, headers=headers) + if res.status_code != 200: + raise Exception( + f'Bulk delete of {bucket_name} failed with status {res.status_code}' + ) + return res + + def head_bucket(self, bucket_name): + """ + Head a container. Throws StorageNoSuchKeyError if it does not exist. + :param bucket_name: name of the container + :return: response headers + """ + url = '/'.join([self.endpoint, bucket_name]) + res = self.session.head(url) + if res.status_code == 404: + raise StorageNoSuchKeyError(bucket_name, '') + if not 200 <= res.status_code < 300: + raise Exception('{} - {}'.format(res.status_code, bucket_name)) + return res.headers def list_objects(self, bucket_name, prefix='', match_pattern=None): """ - Lists the objects in a bucket. + Lists the objects in a bucket. Throws StorageNoSuchKeyError if the bucket does not exist. :param bucket_name: name of the bucket :param prefix: prefix to filter object names :return: one dict per object, with its key ('Key') and size in bytes ('Size') @@ -269,7 +369,19 @@ def list_objects(self, bucket_name, prefix='', match_pattern=None): # A listing returns at most one page of objects (10,000 by default), # and the next one starts after the last name of the previous one while True: - page = self.session.get(url, params=params).json() + res = self.session.get(url, params=params) + if res.status_code == 404: + raise StorageNoSuchKeyError(bucket_name, prefix) + # Past the last page the listing is empty. Some Swift versions + # answer that with a 204 and no body. An empty error response + # is not the end of the listing + if res.status_code == 204 or (res.status_code == 200 and not res.content): + return objects + if res.status_code != 200: + raise Exception( + f'{res.status_code} listing {bucket_name}/{prefix}' + ) + page = res.json() if not page: return objects # Swift names the key 'name' and the size 'bytes' diff --git a/lithops/storage/cloud_proxy.py b/lithops/storage/cloud_proxy.py index c3529ba25..d7ea4b4e0 100644 --- a/lithops/storage/cloud_proxy.py +++ b/lithops/storage/cloud_proxy.py @@ -210,7 +210,7 @@ def exists(self, path): return False key = _to_key(path) dirpath = _to_key(path, as_dir=True) - for k in self._storage.list_bucket_keys(prefix=key): + for k in remove_lithops_keys(self._storage.list_bucket_keys(prefix=key)): if k.startswith(dirpath) or k == key: return True return False diff --git a/lithops/tests/test_call_async.py b/lithops/tests/test_call_async.py index 0b2ec3699..a5ab66fad 100644 --- a/lithops/tests/test_call_async.py +++ b/lithops/tests/test_call_async.py @@ -63,6 +63,33 @@ def test_dict_iterdata(self): result = fexec.get_result() assert result == 10 + def test_single_future_is_a_chain(self): + fexec = lithops.FunctionExecutor(config=pytest.lithops_config) + future = fexec.call_async(lambda x: x + 1, 1) + fexec.call_async(lambda x: x * 3, future) + assert fexec.get_result() == 6 + + def test_list_of_futures_is_one_chained_call(self): + """A list used to be passed through as future objects""" + def double(x): + return x * 2 + + def total(values): + return sum(values) + + fexec = lithops.FunctionExecutor(config=pytest.lithops_config) + futures = fexec.map(double, [1, 2, 3]) + chained = fexec.call_async(total, list(futures)) + assert chained.job_id.startswith('A') + assert fexec.get_result(chained) == 12 + + def test_tuple_of_futures_is_one_chained_call(self): + fexec = lithops.FunctionExecutor(config=pytest.lithops_config) + first = fexec.call_async(lambda x: x + 1, 1) + second = fexec.call_async(lambda x: x + 2, 1) + chained = fexec.call_async(lambda x, y: x * y, (first, second)) + assert fexec.get_result(chained) == 6 + def test_object_with_side_effects(self): se = SideEffect() fexec = lithops.FunctionExecutor(config=pytest.lithops_config) diff --git a/lithops/tests/test_executors.py b/lithops/tests/test_executors.py index d02a3638e..2005155f5 100644 --- a/lithops/tests/test_executors.py +++ b/lithops/tests/test_executors.py @@ -617,6 +617,25 @@ def test_a_failed_wait_honours_clean_jobs(self, mock_wait): clean_jobs=False) cleanup.assert_not_called() + @patch('lithops.executors.wait', side_effect=TimeoutError('late')) + def test_wait_can_return_on_timeout_without_cancelling(self, mock_wait): + """A retrying wait has to see what finished, not have it deleted""" + future = FakeFuture(done=True, success=True) + running = FakeFuture() + executor = _bare_executor(data_cleaner=True) + with patch.object(executor, '_cleanup_jobs') as cleanup: + done, pending = executor.wait( + [future, running], show_progressbar=False, + cancel_on_timeout=False, + ) + cleanup.assert_not_called() + executor.invoker.discard_pending.assert_not_called() + executor.job_monitor.remove.assert_called_once_with([future]) + assert future._exception_set is False + assert running._exception_set is False + assert done == [future] + assert pending == [running] + def test_get_result_takes_a_single_future(self): """The signature takes a ResponseFuture, which has no len()""" future = FakeFuture(_result=42) diff --git a/lithops/tests/test_k8s_backend.py b/lithops/tests/test_k8s_backend.py index 494a9f715..585278d87 100644 --- a/lithops/tests/test_k8s_backend.py +++ b/lithops/tests/test_k8s_backend.py @@ -5,7 +5,9 @@ # the backend creates. # +import base64 import copy +import json from unittest.mock import MagicMock import pytest @@ -34,6 +36,26 @@ def _node(arch): # load_config # --------------------------------------------------------------------------- +class TestRegistrySecret: + + def test_docker_config_uses_the_keys_kubernetes_reads(self): + backend = _make_backend({ + 'docker_user': 'me', + 'docker_password': 'pw', + 'docker_server': 'registry.example', + }) + backend.namespace = 'ns' + backend.core_api.delete_namespaced_secret.side_effect = ApiException(status=404) + backend._create_container_registry_secret() + secret = backend.core_api.create_namespaced_secret.call_args.args[1] + payload = json.loads(base64.b64decode(secret.data['.dockerconfigjson'])) + auth = payload['auths']['registry.example'] + assert auth['username'] == 'me' + assert auth['password'] == 'pw' + assert auth['auth'] == base64.b64encode(b'me:pw').decode() + assert 'Username' not in auth + + class TestLoadConfig: def test_defaults_applied(self): diff --git a/lithops/tests/test_knative_backend.py b/lithops/tests/test_knative_backend.py index f9ac9c82c..7549a14b0 100644 --- a/lithops/tests/test_knative_backend.py +++ b/lithops/tests/test_knative_backend.py @@ -325,6 +325,46 @@ def test_list(self): assert backend.list_runtimes('other') == [('docker.io/user/other:1', 2048, '3.8.1', 'b')] +class TestDeployRuntime: + + def test_a_metadata_failure_deletes_the_service(self): + backend = _make_backend() + backend._get_default_runtime_image_name = MagicMock(return_value='other') + backend._create_container_registry_secret = MagicMock() + backend._create_service = MagicMock() + backend._generate_runtime_meta = MagicMock(side_effect=Exception('metadata down')) + backend.delete_runtime = MagicMock() + with pytest.raises(Exception, match='metadata down'): + backend.deploy_runtime('img', 256, 60) + backend.delete_runtime.assert_called_once_with('img', 256) + + def test_the_metadata_error_survives_a_failed_cleanup(self): + backend = _make_backend() + backend._get_default_runtime_image_name = MagicMock(return_value='other') + backend._create_container_registry_secret = MagicMock() + backend._create_service = MagicMock() + backend._generate_runtime_meta = MagicMock(side_effect=Exception('metadata down')) + backend.delete_runtime = MagicMock(side_effect=Exception('cleanup')) + with pytest.raises(Exception, match='metadata down'): + backend.deploy_runtime('img', 256, 60) + + def test_unknown_kube_context_names_the_known_ones(self, tmp_path): + kubeconfig = tmp_path / 'kubeconfig' + kubeconfig.write_text('apiVersion: v1\n') + contexts = ( + [{'name': 'kind', 'context': {'namespace': 'ns', 'cluster': 'c'}}], + {'name': 'kind', 'context': {'namespace': 'ns', 'cluster': 'c'}}, + ) + with patch.object(knative, 'load_kube_config'), \ + patch.object(knative, 'list_kube_config_contexts', return_value=contexts), \ + patch.object(knative, 'client'): + with pytest.raises(Exception, match="context 'typo' was not found"): + KnativeServingBackend( + {'kubecfg_path': str(kubeconfig), 'kubecfg_context': 'typo'}, + MagicMock(), + ) + + class TestDeleteRuntime: def test_waits_until_gone(self): diff --git a/lithops/tests/test_retries.py b/lithops/tests/test_retries.py index 5d04f4bab..9194a59bf 100644 --- a/lithops/tests/test_retries.py +++ b/lithops/tests/test_retries.py @@ -293,6 +293,62 @@ def test_wait_retries_failed_futures_until_all_complete(self): assert retrying.response_future is retried inner.map.assert_called_once() + def test_wait_shares_one_timeout_across_retries(self, monkeypatch): + """Each retry round used to be given the original timeout again""" + clock = {'now': 1000.0} + monkeypatch.setattr('lithops.retries.time.monotonic', lambda: clock['now']) + failed = FakeResponseFuture(error=True) + retried = FakeResponseFuture(error=False, result=1) + inner = MagicMock() + inner.config = {} + recorded = [] + + def on_wait(fs, **kwargs): + assert kwargs['cancel_on_timeout'] is False + recorded.append(kwargs['timeout']) + clock['now'] += 4 + if len(recorded) == 1: + return [failed], [] + return [retried], [] + + inner.wait.side_effect = on_wait + inner.map.return_value = [retried] + retrying = RetryingFuture(failed, map_function=lambda x: x, input=1, retries=1) + done, pending = RetryingFunctionExecutor(inner).wait( + [retrying], timeout=10, throw_except=False, show_progressbar=False + ) + assert pending == [] + assert done == [retrying] + assert recorded == [10, 6] + + def test_wait_timeout_does_not_retry_past_the_deadline(self, monkeypatch): + """ + A timeout used to cancel every call, so a failure that had already + come back was never retried, and the next round started a new budget + """ + clock = {'now': 1000.0} + monkeypatch.setattr('lithops.retries.time.monotonic', lambda: clock['now']) + failed = FakeResponseFuture(error=True) + still_running = FakeResponseFuture() + inner = MagicMock() + inner.config = {} + + def on_wait(fs, **kwargs): + clock['now'] += kwargs['timeout'] + return [failed], [still_running] + + inner.wait.side_effect = on_wait + futures = [ + RetryingFuture(failed, map_function=lambda x: x, input=1, retries=2), + RetryingFuture(still_running, map_function=lambda x: x, input=2, retries=2), + ] + with pytest.raises(TimeoutError, match='Timeout of 10 seconds'): + RetryingFunctionExecutor(inner).wait( + futures, timeout=10, throw_except=False, show_progressbar=False + ) + inner.map.assert_not_called() + assert inner.wait.call_count == 1 + def test_wait_always_returns_after_first_poll(self): pending_resp = FakeResponseFuture() inner = MagicMock() diff --git a/lithops/tests/test_serverless.py b/lithops/tests/test_serverless.py index 5a278cb96..d6945b2ad 100644 --- a/lithops/tests/test_serverless.py +++ b/lithops/tests/test_serverless.py @@ -360,3 +360,145 @@ def test_other_registries_are_not_checked(self, backend): test_permissions = self._granted(backend, []) backend._ensure_artifact_registry_upload_permission('docker.io/me/runtime:01') test_permissions.assert_not_called() + + +class TestGcpFunctionDeletion: + + def test_only_a_missing_function_ends_the_wait(self): + pytest.importorskip('googleapiclient') + from googleapiclient.errors import HttpError + from lithops.serverless.backends.gcp_functions.gcp_functions import GCPFunctionsBackend + backend = GCPFunctionsBackend.__new__(GCPFunctionsBackend) + backend.num_retries = 0 + backend.retry_sleep = 0 + backend._api_resource = MagicMock() + get = ( + backend._api_resource.projects.return_value + .locations.return_value.functions.return_value.get + ) + + def http_error(status): + return HttpError(MagicMock(status=status, reason='no'), b'no') + + get.return_value.execute.side_effect = http_error(503) + with pytest.raises(HttpError): + backend._wait_function_deleted('projects/p/locations/l/functions/f') + get.return_value.execute.side_effect = http_error(404) + backend._wait_function_deleted('projects/p/locations/l/functions/f') + + +class TestGcpFunctionsDeploy: + + def _backend(self): + from lithops.serverless.backends.gcp_functions.gcp_functions import GCPFunctionsBackend + backend = GCPFunctionsBackend.__new__(GCPFunctionsBackend) + backend.trigger = 'https' + backend.internal_storage = MagicMock() + backend.internal_storage.storage.config = {} + return backend + + def test_runtime_metadata_is_read_once(self): + backend = self._backend() + backend.invoke = MagicMock(return_value={'preinstalls': ['x']}) + assert backend._generate_runtime_meta('rt', 256) == {'preinstalls': ['x']} + backend.invoke.assert_called_once() + + def test_metadata_without_preinstalls_is_not_read_again(self): + backend = self._backend() + backend.invoke = MagicMock(return_value={}) + with pytest.raises(Exception, match='Failed getting runtime metadata'): + backend._generate_runtime_meta('rt', 256) + backend.invoke.assert_called_once() + + def test_a_metadata_failure_deletes_the_function(self, tmp_path, monkeypatch): + from lithops.serverless.backends.gcp_functions import config as gcf_config + zip_path = tmp_path / 'rt.zip' + zip_path.write_bytes(b'zip') + monkeypatch.setattr(gcf_config, 'FH_ZIP_LOCATION', str(tmp_path / '{}.zip')) + backend = self._backend() + backend._get_default_runtime_name = MagicMock(return_value='other') + backend._get_runtime_bin_location = MagicMock(return_value='bin') + backend._create_function = MagicMock() + backend._generate_runtime_meta = MagicMock(side_effect=Exception('metadata down')) + backend.delete_runtime = MagicMock() + with pytest.raises(Exception, match='metadata down'): + backend.deploy_runtime('rt', 256, 60) + backend.delete_runtime.assert_called_once_with('rt', 256) + assert not zip_path.exists() + + def test_the_metadata_error_survives_a_failed_cleanup(self, tmp_path, monkeypatch): + from lithops.serverless.backends.gcp_functions import config as gcf_config + (tmp_path / 'rt.zip').write_bytes(b'zip') + monkeypatch.setattr(gcf_config, 'FH_ZIP_LOCATION', str(tmp_path / '{}.zip')) + backend = self._backend() + backend._get_default_runtime_name = MagicMock(return_value='other') + backend._get_runtime_bin_location = MagicMock(return_value='bin') + backend._create_function = MagicMock() + backend._generate_runtime_meta = MagicMock(side_effect=Exception('metadata down')) + backend.delete_runtime = MagicMock(side_effect=Exception('cleanup')) + with pytest.raises(Exception, match='metadata down'): + backend.deploy_runtime('rt', 256, 60) + + +class TestCloudRunDeployAndDelete: + + def _backend(self): + pytest.importorskip('googleapiclient') + from lithops.serverless.backends.gcp_cloudrun.cloudrun import GCPCloudRunBackend + backend = GCPCloudRunBackend.__new__(GCPCloudRunBackend) + backend.project_name = 'proj' + backend._api_resource = MagicMock() + return backend + + @staticmethod + def _http_error(status): + from googleapiclient.errors import HttpError + return HttpError(MagicMock(status=status, reason='no'), b'no') + + def test_a_metadata_failure_deletes_the_service(self): + backend = self._backend() + backend._get_default_runtime_image_name = MagicMock(return_value='other') + backend._create_service = MagicMock() + backend._generate_runtime_meta = MagicMock(side_effect=Exception('metadata down')) + backend.delete_runtime = MagicMock() + with pytest.raises(Exception, match='metadata down'): + backend.deploy_runtime('docker.io/me/runtime:1', 256, 60) + backend.delete_runtime.assert_called_once_with('docker.io/me/runtime:1', 256) + + def test_the_metadata_error_survives_a_failed_cleanup(self): + backend = self._backend() + backend._get_default_runtime_image_name = MagicMock(return_value='other') + backend._create_service = MagicMock() + backend._generate_runtime_meta = MagicMock(side_effect=Exception('metadata down')) + backend.delete_runtime = MagicMock(side_effect=Exception('cleanup')) + with pytest.raises(Exception, match='metadata down'): + backend.deploy_runtime('docker.io/me/runtime:1', 256, 60) + + def test_only_a_missing_service_ends_the_wait(self): + backend = self._backend() + services = backend._api_resource.namespaces.return_value.services.return_value + services.delete.return_value.execute.return_value = {} + services.get.return_value.execute.side_effect = self._http_error(503) + with pytest.raises(Exception): + backend._delete_service('svc') + services.get.return_value.execute.side_effect = [ + {}, + self._http_error(404), + ] + with patch('lithops.serverless.backends.gcp_cloudrun.cloudrun.time.sleep'): + backend._delete_service('svc') + + def test_a_missing_service_is_already_gone(self): + backend = self._backend() + services = backend._api_resource.namespaces.return_value.services.return_value + services.delete.return_value.execute.side_effect = self._http_error(404) + backend._delete_service('svc') + services.get.assert_not_called() + + def test_a_failed_delete_is_raised(self): + backend = self._backend() + services = backend._api_resource.namespaces.return_value.services.return_value + services.delete.return_value.execute.side_effect = self._http_error(403) + with pytest.raises(Exception): + backend._delete_service('svc') + services.get.assert_not_called() diff --git a/lithops/tests/test_standalone.py b/lithops/tests/test_standalone.py index f7e6b1a21..7e21d0a0e 100644 --- a/lithops/tests/test_standalone.py +++ b/lithops/tests/test_standalone.py @@ -531,6 +531,37 @@ def test_aws_deletes_a_local_ssh_key_that_could_not_be_imported(self, tmp_path): ) assert list(ssh_dir.iterdir()) == [] + def test_ibm_deletes_a_floating_ip_it_allocated(self): + backend, load_data, _, _, _, _ = _ibm_backend_for_init() + backend.config['vpc_id'] = 'r006-vpc123456' + backend.config['zone_name'] = 'eu-de-1' + backend.vpc_cli.list_floating_ips.return_value.get_result.return_value = { + 'floating_ips': [] + } + backend.vpc_cli.create_floating_ip.return_value.result = { + 'address': '1.2.3.4', 'id': 'fip-new' + } + with patch.object(backend, '_request_image_id', side_effect=Exception('quota')): + self._failed_init( + backend, load_data, + '_create_vpc', '_create_ssh_key', '_create_subnet', '_create_gateway', + ) + backend.vpc_cli.delete_floating_ip.assert_called_once_with(id='fip-new') + + def test_aws_spot_price_without_history_raises(self): + from lithops.standalone.backends.aws_ec2.aws_ec2 import AWSEC2Backend + backend = AWSEC2Backend.__new__(AWSEC2Backend) + backend.config = { + 'request_spot_instances': True, + 'worker_instance_type': 't2.micro', + } + backend.ec2_client = MagicMock() + backend.ec2_client.describe_spot_price_history.return_value = { + 'SpotPriceHistory': [] + } + with pytest.raises(ValueError, match='No spot price history for worker instance type t2.micro'): + backend._request_spot_price() + def test_ibm_deletes_a_local_ssh_key_that_could_not_be_registered(self, tmp_path): backend, load_data, _, _, _, deletes = _ibm_backend_for_init() from ibm_cloud_sdk_core import ApiException diff --git a/lithops/tests/test_storage_layer.py b/lithops/tests/test_storage_layer.py index a012c66d0..be4a9cf44 100644 --- a/lithops/tests/test_storage_layer.py +++ b/lithops/tests/test_storage_layer.py @@ -258,6 +258,106 @@ def test_generate_bucket_name(self): assert backend.generate_bucket_name() == 'lithops-reg-abcdef' assert config['storage_bucket'] == 'lithops-reg-abcdef' + def _token_response(self, url='https://swift.example/v1/AUTH_abc', region='RegionOne'): + catalog = { + 'token': { + 'project': {'id': 'a' * 32}, + 'catalog': [{ + 'type': 'object-store', + 'name': 'swift', + 'endpoints': [ + {'interface': 'internal', 'region': region, 'url': 'http://internal'}, + {'interface': 'public', 'region': region, 'region_id': region, 'url': url}, + ], + }], + } + } + response = MagicMock(status_code=201, headers={'X-Subject-Token': 'tok'}) + response.json.return_value = catalog + return response + + def test_keystone_scopes_a_project_by_name_and_keeps_https(self): + from lithops.storage.backends.swift.swift import StorageBackend + response = self._token_response() + with patch('lithops.storage.backends.swift.swift.requests.post', return_value=response) as posted: + backend = StorageBackend({ + 'auth_url': 'http://keystone:5000/v3/', + 'user_name': 'admin', + 'project_name': 'admin', + 'password': 'secret', + 'region': 'RegionOne', + }) + assert backend.endpoint == 'https://swift.example/v1/AUTH_abc' + sent = json.loads(posted.call_args.kwargs['data']) + assert posted.call_args.args[0] == 'http://keystone:5000/v3/auth/tokens' + assert sent['auth']['scope']['project'] == { + 'name': 'admin', 'domain': {'name': 'Default'}, + } + assert sent['auth']['identity']['password']['user']['name'] == 'admin' + assert 'id' not in sent['auth']['scope']['project'] + + def test_keystone_scopes_an_id_without_a_domain(self): + from lithops.storage.backends.swift.swift import StorageBackend + user_id = 'f' * 32 + project_id = 'a' * 32 + with patch( + 'lithops.storage.backends.swift.swift.requests.post', + return_value=self._token_response('http://0.0.0.0:8080/v1/AUTH_%(project_id)s'), + ) as posted: + backend = StorageBackend({ + 'auth_url': 'http://127.0.0.1:5000', + 'user_id': user_id, + 'project_id': project_id, + 'password': 'secret', + 'region': 'RegionOne', + }) + sent = json.loads(posted.call_args.kwargs['data']) + assert sent['auth']['identity']['password']['user'] == { + 'id': user_id, 'password': 'secret', + } + assert sent['auth']['scope']['project'] == {'id': project_id} + assert backend.endpoint == f'http://127.0.0.1:8080/v1/AUTH_{project_id}' + + def test_an_unknown_region_names_the_known_ones(self): + from lithops.storage.backends.swift.swift import StorageBackend + with patch( + 'lithops.storage.backends.swift.swift.requests.post', + return_value=self._token_response(), + ): + with pytest.raises(Exception, match='region \'nope\''): + StorageBackend({ + 'auth_url': 'http://keystone:5000', + 'user_name': 'admin', + 'project_name': 'admin', + 'password': 'secret', + 'region': 'nope', + }) + + def test_keystone_error_message_is_raised(self): + from lithops.storage.backends.swift.swift import StorageBackend + response = MagicMock(status_code=401, text='no') + response.json.return_value = {'error': {'message': 'The request you have made requires authentication.'}} + with patch('lithops.storage.backends.swift.swift.requests.post', return_value=response): + with pytest.raises(Exception, match='401 - The request you have made requires authentication'): + StorageBackend({ + 'auth_url': 'http://keystone:5000', + 'user_name': 'admin', + 'project_name': 'admin', + 'password': 'secret', + 'region': 'RegionOne', + }) + + def test_create_bucket_accepts_an_existing_container(self): + from lithops.storage.backends.swift.swift import StorageBackend + backend = StorageBackend.__new__(StorageBackend) + backend.endpoint = 'http://swift/v1/AUTH_x' + backend.session = MagicMock() + backend.session.put.return_value.status_code = 202 + backend.create_bucket('lithops') + assert backend.session.put.call_args.args[0] == 'http://swift/v1/AUTH_x/lithops' + backend.session.put.return_value.status_code = 500 + with pytest.raises(Exception, match='PUT container lithops failed'): + backend.create_bucket('lithops') def test_get_object_missing_key(self): from lithops.storage.backends.swift.swift import StorageBackend @@ -275,6 +375,8 @@ def test_list_errors_propagate(self): backend.endpoint = 'http://swift' backend.session = MagicMock() page = [{'name': 'p/a', 'bytes': 1}, {'name': 'p/b', 'bytes': 2}] + backend.session.get.return_value.status_code = 200 + backend.session.get.return_value.content = b'page' backend.session.get.return_value.json.side_effect = [page, []] assert backend.list_keys('b', 'p/') == ['p/a', 'p/b'] backend.session.get.side_effect = ValueError('boom') @@ -287,6 +389,7 @@ def _listing_backend(): backend = StorageBackend.__new__(StorageBackend) backend.endpoint = 'http://swift' backend.session = MagicMock() + backend.session.get.return_value.status_code = 200 backend.session.get.return_value.json.side_effect = [[ {'name': f'p/{n}', 'bytes': size, 'hash': 'h', 'content_type': 'text/csv', 'last_modified': '2024-01-01T00:00:00.000000'} @@ -311,6 +414,65 @@ def test_list_objects_pages_past_the_listing_limit(self): assert backend.session.get.call_count == 4 assert backend.session.get.call_args.kwargs['params']['marker'] == 'k2' + def test_list_objects_ends_at_an_empty_listing_with_no_body(self): + """Some Swift versions answer the request past the last page with a 204 and no body""" + backend = self._listing_backend() + first, empty = MagicMock(status_code=200), MagicMock(status_code=204, content=b'') + first.json.return_value = [{'name': 'k0', 'bytes': 1}] + empty.json.side_effect = ValueError('no JSON body') + backend.session.get.side_effect = [first, empty] + assert backend.list_objects('b') == [{'Key': 'k0', 'Size': 1}] + + def test_list_objects_of_a_missing_bucket(self): + backend = self._listing_backend() + backend.session.get.return_value.status_code = 404 + with pytest.raises(StorageNoSuchKeyError): + backend.list_objects('b', 'p/') + + def test_list_objects_does_not_treat_an_empty_error_as_the_end(self): + backend = self._listing_backend() + backend.session.get.return_value.status_code = 500 + backend.session.get.return_value.content = b'' + with pytest.raises(Exception, match='500 listing'): + backend.list_objects('b') + + def test_put_object_raises_on_failure(self): + from lithops.storage.backends.swift.swift import StorageBackend + backend = StorageBackend.__new__(StorageBackend) + backend.endpoint = 'http://swift' + backend.session = MagicMock() + backend.session.put.return_value.status_code = 500 + with pytest.raises(Exception, match='PUT k failed'): + backend.put_object('b', 'k', b'data') + backend.session.put.side_effect = ConnectionError('down') + with pytest.raises(ConnectionError): + backend.put_object('b', 'k', b'data') + + def test_get_and_head_keep_non_404_errors(self): + from lithops.storage.backends.swift.swift import StorageBackend + backend = StorageBackend.__new__(StorageBackend) + backend.endpoint = 'http://swift' + backend.token = 't' + backend.session = MagicMock() + backend.session.get.return_value.status_code = 503 + with pytest.raises(Exception, match='503'): + backend.get_object('b', 'k') + backend.session.head.return_value.status_code = 503 + with pytest.raises(Exception, match='503'): + backend.head_object('b', 'k') + + def test_head_bucket(self): + from lithops.storage.backends.swift.swift import StorageBackend + backend = StorageBackend.__new__(StorageBackend) + backend.endpoint = 'http://swift' + backend.session = MagicMock() + backend.session.head.return_value.status_code = 204 + backend.session.head.return_value.headers = {'X-Container-Object-Count': '1'} + assert backend.head_bucket('b')['X-Container-Object-Count'] == '1' + backend.session.head.return_value.status_code = 404 + with pytest.raises(StorageNoSuchKeyError): + backend.head_bucket('missing') + def test_list_objects_match_pattern(self): """The glob pattern failed on the raw entries, which have no 'Key'""" storage = _bare_storage('swift') @@ -354,6 +516,75 @@ def test_list_objects_skips_keys_gone_since_listed(self, backend): backend.infinispan_client.get.return_value.content = b'["p/a", "p/gone"]' assert backend.list_objects('b') == [{'Key': 'p/a', 'Size': 1}] + def test_get_object_range_and_head_bucket(self): + from lithops.storage.backends.infinispan import StorageBackend + backend = StorageBackend.__new__(StorageBackend) + backend.endpoint = 'http://ispn' + backend.auth = None + backend.headers = {} + backend.infinispan_client = MagicMock() + backend.infinispan_client.get.return_value = MagicMock( + status_code=200, content=b'abcdefghij' + ) + assert backend.get_object('b', 'k', extra_get_args={'Range': 'bytes=2-5'}) == b'cdef' + assert backend.get_object('b', 'k', extra_get_args={'Range': 'bytes=-3'}) == b'hij' + backend.infinispan_client.head.return_value = MagicMock(status_code=404) + with pytest.raises(StorageNoSuchKeyError): + backend.head_bucket('b') + backend.infinispan_client.head.return_value = MagicMock(status_code=200) + assert backend.head_bucket('b')['ResponseMetadata']['HTTPStatusCode'] == 200 + + def test_put_object_raises_on_http_error(self): + import requests + from lithops.storage.backends.infinispan import StorageBackend + backend = StorageBackend.__new__(StorageBackend) + backend.endpoint = 'http://ispn' + backend.auth = None + backend.headers = {} + backend.infinispan_client = MagicMock() + backend.infinispan_client.put.return_value.raise_for_status.side_effect = ( + requests.HTTPError('500') + ) + with pytest.raises(requests.HTTPError): + backend.put_object('b', 'k', b'data') + + +class TestRedisBackend: + + def _backend(self): + pytest.importorskip('redis') + from lithops.storage.backends.redis import StorageBackend + backend = StorageBackend.__new__(StorageBackend) + backend._client = MagicMock() + return backend + + def test_range_read_distinguishes_a_missing_key(self): + backend = self._backend() + pipe = backend._client.pipeline.return_value + pipe.execute.return_value = (True, b'cdef') + assert backend.get_object('b', 'k', extra_get_args={'Range': 'bytes=2-5'}) == b'cdef' + pipe.getrange.assert_called_with('b/k', 2, 5) + pipe.execute.return_value = (False, b'') + with pytest.raises(StorageNoSuchKeyError): + backend.get_object('b', 'k', extra_get_args={'Range': 'bytes=0-1'}) + + def test_open_ended_and_suffix_ranges(self): + backend = self._backend() + pipe = backend._client.pipeline.return_value + pipe.execute.return_value = (True, b'cdef') + backend.get_object('b', 'k', extra_get_args={'Range': 'bytes=2-'}) + pipe.getrange.assert_called_with('b/k', 2, -1) + backend.get_object('b', 'k', extra_get_args={'Range': 'bytes=-3'}) + pipe.getrange.assert_called_with('b/k', -3, -1) + + def test_head_bucket(self): + backend = self._backend() + backend._client.exists.return_value = 0 + with pytest.raises(StorageNoSuchKeyError): + backend.head_bucket('b') + backend._client.exists.return_value = 1 + assert backend.head_bucket('b')['ResponseMetadata']['HTTPStatusCode'] == 200 + class TestCloudBackendRanges: """Open ranges work, and the caller's extra_get_args is not modified""" @@ -430,6 +661,110 @@ def page(names, next_start): assert [o['Key'] for o in backend.list_objects('bkt', 'p')] == ['a', 'b', 'c'] assert backend.os_client.list_objects.call_args.kwargs['start'] == 'c' + def test_oracle_list_keeps_non_404_errors(self): + import oci + pytest.importorskip('oci') + from lithops.storage.backends.oracle_oss import StorageBackend + backend = StorageBackend.__new__(StorageBackend) + backend.namespace = 'ns' + backend.os_client = MagicMock() + backend.os_client.list_objects.side_effect = oci.exceptions.ServiceError( + 500, 'InternalError', {}, 'down' + ) + with pytest.raises(oci.exceptions.ServiceError): + backend.list_objects('bkt', 'p') + backend.os_client.list_objects.side_effect = oci.exceptions.ServiceError( + 404, 'BucketNotFound', {}, 'missing' + ) + with pytest.raises(StorageNoSuchKeyError): + backend.list_objects('bkt', 'p') + + +class TestS3MissingBucket: + + @staticmethod + def _cases(): + found = [] + try: + import botocore + from lithops.storage.backends.aws_s3 import StorageBackend as AWS + found.append((AWS, botocore.exceptions.ClientError)) + except ImportError: + pass + try: + import ibm_botocore + from lithops.storage.backends.ibm_cos import StorageBackend as IBM + found.append((IBM, ibm_botocore.exceptions.ClientError)) + except ImportError: + pass + try: + import botocore + from lithops.storage.backends.ceph import StorageBackend as Ceph + found.append((Ceph, botocore.exceptions.ClientError)) + except ImportError: + pass + try: + import botocore + from lithops.storage.backends.minio import StorageBackend as Minio + found.append((Minio, botocore.exceptions.ClientError)) + except ImportError: + pass + if not found: + pytest.skip('no S3-compatible storage backend is installed') + return found + + def test_list_maps_no_such_bucket_and_keeps_other_errors(self): + for cls, error in self._cases(): + backend = cls.__new__(cls) + client = MagicMock() + backend.s3_client = client + backend.cos_client = client + + def boom(code, error=error): + client.get_paginator.return_value.paginate.side_effect = error( + {'Error': {'Code': code, 'Message': code}}, 'ListObjectsV2' + ) + + boom('NoSuchBucket') + with pytest.raises(StorageNoSuchKeyError): + backend.list_objects('missing') + boom('NoSuchBucket') + with pytest.raises(StorageNoSuchKeyError): + backend.list_keys('missing') + boom('AccessDenied') + with pytest.raises(error): + backend.list_objects('missing') + + +class TestAzureAndGcpMissingObjects: + + def test_azure_head_object_maps_a_missing_blob(self): + pytest.importorskip('azure.storage.blob') + from azure.core.exceptions import ResourceNotFoundError + from lithops.storage.backends.azure_storage import StorageBackend + backend = StorageBackend.__new__(StorageBackend) + backend.blob_client = MagicMock() + blob = backend.blob_client.get_container_client.return_value.get_blob_client.return_value + blob.get_blob_properties.side_effect = ResourceNotFoundError('gone') + with pytest.raises(StorageNoSuchKeyError): + backend.head_object('b', 'k') + blob.get_blob_properties.side_effect = ConnectionError('down') + with pytest.raises(ConnectionError): + backend.head_object('b', 'k') + + def test_gcp_head_bucket_maps_not_found(self): + pytest.importorskip('google.cloud.storage') + from google.api_core import exceptions as google_exceptions + from lithops.storage.backends.gcp_storage import StorageBackend + backend = StorageBackend.__new__(StorageBackend) + backend.client = MagicMock() + backend.client.get_bucket.side_effect = google_exceptions.NotFound('gone') + with pytest.raises(StorageNoSuchKeyError): + backend.head_bucket('missing') + backend.client.get_bucket.side_effect = google_exceptions.Forbidden('no') + with pytest.raises(google_exceptions.Forbidden): + backend.head_bucket('missing') + class TestStorageListObjectsGlob: """ @@ -712,6 +1047,13 @@ def test_listdir_and_path_helpers(self): assert proxy.path.exists('dir/a.txt') is True assert proxy.path.exists('missing') is False + def test_exists_ignores_lithops_keys(self): + fake = FakeCloudStorage(keys=[f'{JOBS_PREFIX}/hidden']) + path = _path(fake) + assert path.isdir(JOBS_PREFIX) is False + assert path.exists(JOBS_PREFIX) is False + assert path.exists(f'{JOBS_PREFIX}/hidden') is False + def test_exists_strips_the_leading_slash_like_isfile(self): # isfile and isdir dropped the leading slash and exists did not, so # '/k' was a file that did not exist diff --git a/lithops/tests/test_wait.py b/lithops/tests/test_wait.py index 3ab4db85a..830be53c1 100644 --- a/lithops/tests/test_wait.py +++ b/lithops/tests/test_wait.py @@ -570,3 +570,6 @@ def test_explicit_futures_count_the_ones_already_done(self): def test_get_result_of_no_futures_is_empty(): assert get_result([], show_progressbar=False) == [] + # None is the default. Wrapping it first made a one-element list + assert get_result(None, show_progressbar=False) == [] + assert get_result(show_progressbar=False) == [] diff --git a/lithops/tests/test_worker.py b/lithops/tests/test_worker.py index b9338d528..9fa9ec6a5 100644 --- a/lithops/tests/test_worker.py +++ b/lithops/tests/test_worker.py @@ -1511,6 +1511,30 @@ def test_fill_optional_args_future_chaining(self): assert data['x'] == 9 assert 'future' not in data + def test_fill_optional_args_list_of_futures_is_one_argument(self): + def total(values): + return sum(values) + + jr = self._runner(_echo, {'x': 1}) + first, second = MagicMock(spec=ResponseFuture), MagicMock(spec=ResponseFuture) + first.result.return_value = 2 + second.result.return_value = 3 + data = {'future': [first, second]} + jr._fill_optional_args(total, data) + assert data == {'values': [2, 3]} + + def test_fill_optional_args_tuple_of_futures_is_unpacked(self): + def mul(x, y): + return x * y + + jr = self._runner(_echo, {'x': 1}) + first, second = MagicMock(spec=ResponseFuture), MagicMock(spec=ResponseFuture) + first.result.return_value = 2 + second.result.return_value = 3 + data = {'future': (first, second)} + jr._fill_optional_args(mul, data) + assert data == {'x': 2, 'y': 3} + def test_fill_optional_args_param_named_future_is_not_chaining(self): def f(future): return future diff --git a/lithops/utils.py b/lithops/utils.py index c3f283344..36e833ef5 100644 --- a/lithops/utils.py +++ b/lithops/utils.py @@ -713,6 +713,11 @@ def _user_signature(func) -> inspect.Signature: return func_sig.replace(parameters=user_parameters) +# call_async() is one activation. A list or tuple of futures is that +# activation's input, resolved on the worker, not one activation per future +_CALL_ASYNC_CHAIN = '_lithops_call_async_chain' + + def _chained_futures(iterdata): """ The futures of a previous job used as the input of this one, or None @@ -797,6 +802,9 @@ def verify_args(func, iterdata, extra_args): new_data = [] for elem in data: + if isinstance(elem, dict) and set(elem) == {_CALL_ASYNC_CHAIN}: + new_data.append({'future': elem[_CALL_ASYNC_CHAIN]}) + continue if isinstance(elem, dict): if has_var_keyword or required <= set(elem): new_data.append(elem) diff --git a/lithops/wait.py b/lithops/wait.py index fbdec6584..ad5a6ab80 100644 --- a/lithops/wait.py +++ b/lithops/wait.py @@ -333,9 +333,11 @@ def get_result( :return: The result of the future/s """ - fs = _as_future_list(fs) + # None is the default and, like wait(), an empty input. Wrapping it + # first made it a one-element list and then crashed on the future if not fs: return [] + fs = _as_future_list(fs) prefix = log_prefix(fs[0].executor_id) logger.info( diff --git a/lithops/worker/jobrunner.py b/lithops/worker/jobrunner.py index 609bf367a..96900ef4f 100644 --- a/lithops/worker/jobrunner.py +++ b/lithops/worker/jobrunner.py @@ -46,7 +46,7 @@ from lithops.future import ResponseFuture from lithops.utils import ( WrappedStreamingBody, sizeof_fmt, is_object_processing_function, - FuturesList, verify_args, WrappedStreamingBodyPartition + FuturesList, verify_args, _chained_futures, WrappedStreamingBodyPartition ) from lithops.storage.utils import create_output_key @@ -166,15 +166,13 @@ def _fill_optional_args( """ func_sig = inspect.signature(function) - if len(data) == 1 and isinstance(data.get('future'), ResponseFuture): - # Function chaining feature. A param of the user function can be - # named future too, so the value tells a chained call apart - out = [ - data.pop('future').result( - internal_storage=self.internal_storage - ) - ] - data.update(verify_args(function, out, None)[0]) + if len(data) == 1: + # Function chaining. A param of the user function can be named + # future too, so only a future (or a list or tuple of them) + # is a chain. A list arrives as one argument of results; a + # tuple is unpacked into the signature, as call_async() does + # with plain data + self._resolve_chained_input(function, data) if 'ibm_cos' in func_sig.parameters: data['ibm_cos'] = self._create_ibm_cos_client() @@ -188,6 +186,28 @@ def _fill_optional_args( if 'id' in func_sig.parameters: data['id'] = int(self.job.call_id) + def _resolve_chained_input(self, function: Callable, data: Dict[str, Any]) -> None: + """Replaces a chained future input with the results it produced""" + chained = data.get('future') + storage = self.internal_storage + + def results_of(futures): + return [ + fut.result(internal_storage=storage) for fut in futures + ] + + if isinstance(chained, ResponseFuture): + resolved = [chained.result(internal_storage=storage)] + elif isinstance(chained, tuple) and _chained_futures(chained) is not None: + resolved = [tuple(results_of(chained))] + elif isinstance(chained, list) and _chained_futures(chained) is not None: + resolved = [results_of(chained)] + else: + return + + data.clear() + data.update(verify_args(function, resolved, None)[0]) + def _wait_futures(self, data: Dict[str, Any]) -> None: """ Replaces the futures a reduce function receives by their results, From a7cf513b63445bd67b9f1862071023afad0eb450 Mon Sep 17 00:00:00 2001 From: JosepSampe Date: Mon, 19 Oct 2026 21:02:37 +0200 Subject: [PATCH 02/13] Fix backends --- CHANGELOG.md | 14 + docs/source/compute_config/aws_lambda.md | 28 +- docs/source/compute_config/azure_functions.md | 2 +- docs/source/compute_config/vm.md | 2 +- docs/source/data_processing.rst | 6 +- lithops/config.py | 2 +- lithops/invokers.py | 5 +- lithops/job/partitioner.py | 92 +-- lithops/scripts/cli.py | 1 + .../backends/aliyun_fc/aliyun_fc.py | 3 +- .../backends/aws_batch/aws_batch.py | 48 +- .../serverless/backends/aws_batch/config.py | 5 +- .../backends/aws_lambda/aws_lambda.py | 204 ++++--- .../serverless/backends/aws_lambda/config.py | 18 + .../azure_containers/azure_containers.py | 4 +- .../azure_functions/azure_functions.py | 85 +-- .../backends/code_engine/code_engine.py | 16 +- .../serverless/backends/code_engine/config.py | 2 + lithops/serverless/backends/ibm_cf/ibm_cf.py | 47 +- .../backends/openwhisk/openwhisk.py | 13 +- .../serverless/backends/oracle_f/oracle_f.py | 36 +- .../backends/singularity/singularity.py | 1 + .../backends/azure_vms/azure_vms.py | 14 +- .../standalone/backends/azure_vms/config.py | 2 +- lithops/standalone/backends/vm/vm.py | 44 +- lithops/standalone/master.py | 23 +- lithops/standalone/standalone.py | 68 ++- lithops/tests/test_aws_backends.py | 522 ++++++++++++++++++ .../test_azure_aliyun_oracle_backends.py | 310 +++++++++++ lithops/tests/test_config.py | 7 + lithops/tests/test_ibm_openwhisk_backends.py | 214 +++++++ lithops/tests/test_invokers.py | 28 +- lithops/tests/test_job.py | 160 +++++- lithops/tests/test_scripts.py | 18 + lithops/tests/test_standalone.py | 293 ++++++++++ lithops/tests/test_standalone_master.py | 45 ++ 36 files changed, 2118 insertions(+), 264 deletions(-) create mode 100644 lithops/tests/test_aws_backends.py create mode 100644 lithops/tests/test_azure_aliyun_oracle_backends.py create mode 100644 lithops/tests/test_ibm_openwhisk_backends.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 78a08dcfb..5be0e2f61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,20 @@ - [Monitoring] Worker tokens of jobs dropped by `wait()`, replaced monitors and restarted invokers are handed back once, a held `__end__` is no longer replaced by an `__init__`, and the workers of a job that owes no token are forgotten instead of kept for as long as the job monitor runs; the shared RabbitMQ connection of a worker is closed at exit. - [Concurrent] `wait()` and `as_completed()` accept generators. - [Core] An executor that fails to be created no longer logs a misleading `job_monitor` error at exit. +- [Core] The FaaS invoker no longer runs up to two workers beyond `max_workers`. `obj_chunk_number` splits an object into exactly that many chunks, or fewer non-empty ones when it is smaller. URL inputs are sized following redirects, an HTTP error raises, and only `Accept-Ranges: bytes` enables partitioning. A local input path that does not exist raises `FileNotFoundError`, and local inputs with no files raise like an empty bucket. A mode section with no body in the config file no longer crashes config loading. +- [CLI] `lithops runtime update` stores `runtime_timeout` in the runtime metadata, which every later job needed. +- [Standalone] `get_result()` on one job no longer cancels the other jobs of the executor, and create mode stops its workers only once none of its jobs is left. Worker VMs whose setup fails, and the workers of a job whose master cannot be set up, are stopped instead of left running. Requests to the master from inside a worker send their body, and an error from the master raises. The `vm` backend runs jobs: it reads `ssh_username` and provides what the master and its budget keeper need. Azure VMs are deallocated when stopped, so they are no longer billed, the home directory follows `ssh_username`, and `delete_on_dismantle` defaults to `False` as documented. +- [AWS Lambda] Invocations retry connection errors a few times, re-signing each attempt, instead of retrying any error for ever. `lithops runtime update` updates the code, configuration and tags of a deployed function, and the default runtime gets the `user_tags` too. `env_vars` and `user_tags` accept a `{NAME: VALUE}` mapping or a list of `{name, value}`. Layers with more than 50 versions are deleted. Runtimes and layers past the first page of the listing are found by `lithops runtime list` and `clean`, and deleting a runtime whose function is already gone works. +- [AWS Batch] Maps over more than 20 calls no longer fail on a fractional `max_workers`. Job definitions use the requested `runtime_memory`, checked against the Fargate combinations, and their names carry the requested version, so `lithops runtime update` and `delete` find them, also past the first page of job definitions. Deleting a runtime whose job definition is already gone works. +- [IBM CF] A 404 on invoke redeploys the runtime instead of deadlocking every invoker thread. +- [IBM CF] [OpenWhisk] A failure to read runtime metadata raises that error instead of a `TypeError`. +- [OpenWhisk] An invoke rejected with 401 or 404 raises instead of being reported as an activation id, which made `wait()` hang. +- [Code Engine] `clear()` deletes every job run and config map of the executor, a leftover config map is replaced instead of failing every deploy, and waiting for a job run or a job definition deletion times out. +- [Singularity] Runtime metadata that never arrives raises the intended timeout error. +- [Azure Functions] An invoke that returns an HTTP error other than 429 raises. Waiting for runtime metadata with the pub/sub trigger times out after `runtime_timeout`. `runtime_memory` is honoured after the first deploy: each Flex instance memory size gets its own function app, so existing runtimes are deployed again once per size. `lithops runtime update` and `delete` find a runtime by its name. +- [Azure Container Apps] `lithops runtime update` and `delete` find a runtime by its image, and `list_runtimes` reports memory as a number. +- [Aliyun FC] Retries while reading runtime metadata resend the payload instead of an empty body. +- [Oracle Functions] Calls run on the function of their own runtime and memory, not on the function of the job started last. - [Knative] Calls longer than 5 minutes are no longer cut off by Knative, a load balancer ingress is reached at its own port and may be a hostname, `https` service URLs work, a failed call is reported instead of being invoked again for ever, and a service that never gets ready raises an error. ## [v3.8.0] diff --git a/docs/source/compute_config/aws_lambda.md b/docs/source/compute_config/aws_lambda.md index c08a84349..476da92e6 100644 --- a/docs/source/compute_config/aws_lambda.md +++ b/docs/source/compute_config/aws_lambda.md @@ -104,14 +104,38 @@ In summary, you can use one of the following settings: | aws_lambda | remote_invoker | False | no | Activate the remote invoker feature that uses one cloud function to spawn all the actual `map()` activations | | aws_lambda | architecture | x86_64 | no | Runtime architecture. One of **x86_64** or **arm64** | | aws_lambda | ephemeral_storage | 512 | no | Ephemeral storage (`/tmp`) size in MB (must be between 512 MB and 10240 MB) | -| aws_lambda | user_tags | {} | no | List of {name: ..., value: ...} pairs for Lambda instance user tags | -| aws_lambda | env_vars | {} | no | List of {name: ..., value: ...} pairs for Lambda instance environment variables | +| aws_lambda | user_tags | {} | no | `{NAME: VALUE}` pairs for Lambda instance user tags (container runtimes only). A list of `{name: ..., value: ...}` pairs is also accepted | +| aws_lambda | env_vars | {} | no | `{NAME: VALUE}` pairs for Lambda instance environment variables. A list of `{name: ..., value: ...}` pairs is also accepted | | aws_lambda | namespace | | no | Virtual namespace. This can be useful to virtually group Lithops function workers. The functions deployed by lithops will be prefixed by this namespace. For example you can set it to differentiate between `prod`, `dev` and `stage` environments. | | aws_lambda | runtime_include_function | False | no | If set to true, Lithops will automatically build a new runtime, including the function's code, instead of transferring it through the storage backend at invocation time. This is useful when the function's code size is large (on the order of tens of MB) and the code does not change frequently | ## Additional configuration +### Environment variables and user tags +To set environment variables or user tags on the Lithops lambdas, add them to the `aws_lambda` configuration section: + +```yaml +aws_lambda: + execution_role: + region: + env_vars: + MY_VAR: my-value + user_tags: + project: my-project +``` + +The list form used in previous versions of this page is also accepted: + +```yaml +aws_lambda: + env_vars: + - name: MY_VAR + value: my-value +``` + +User tags are set when a function is created. Run `lithops runtime update` after changing the environment variables so they are applied to the already deployed functions. + ### VPC To connect the Lithops lambda to a VPC, add the following configuration to the `aws_lambda` configuration section: diff --git a/docs/source/compute_config/azure_functions.md b/docs/source/compute_config/azure_functions.md index 9d3d1029d..5dff442df 100644 --- a/docs/source/compute_config/azure_functions.md +++ b/docs/source/compute_config/azure_functions.md @@ -87,7 +87,7 @@ az login |---|---|---|---|---| |azure_functions| resource_group | |no | Name of a resource group, for example: `LithopsResourceGroup`. Lithops will use the `resource_group` set under the `azure` section if it is not set here | |azure_functions| region | |no | The Flex Consumption plan location for the runtime. Use `az functionapp list-flexconsumption-locations` to view the available locations. For example: `westeurope`, `westus2`, etc. Lithops will use the `region` set under the `azure` section if it is not set here| -|azure_functions | runtime_memory | 2048 | no | Flex Consumption instance memory in MB. Supported values: `512`, `2048`, `4096`. Other values are mapped to the nearest supported size | +|azure_functions | runtime_memory | 2048 | no | Flex Consumption instance memory in MB. Supported values: `512`, `2048`, `4096`. Other values are mapped to the nearest supported size. Each instance memory size is deployed as a separate function app | |azure_functions | max_workers | 1000 | no | Max number of parallel workers. Although Azure limits the number of workers to 200, it is convenient to keep this value high| |azure_functions | worker_processes | 1 | no | Lithops-side parallelism setting. Not applied as an Azure Functions app setting on Flex Consumption | |azure_functions| runtime | |no | Runtime name already deployed in the service| diff --git a/docs/source/compute_config/vm.md b/docs/source/compute_config/vm.md index 2e3a87aa7..2448abc54 100644 --- a/docs/source/compute_config/vm.md +++ b/docs/source/compute_config/vm.md @@ -53,7 +53,7 @@ In this backend, you can use any Docker image that contains all the required dep |vm | ssh_password | | no | SSH password for accessing the VM | |vm | ssh_key_filename | | no | Path to SSH key | |vm | runtime | python3 |no | `python3` or a Docker image name | -|vm | worker_processes | 1 | no | Number of Lithops processes within the VM. This can be used to parallelize function activations within the VM. It is recommended to set it to the same number of CPUs as the VM | +|vm | worker_processes | AUTO | no | Number of Lithops processes within the VM. This can be used to parallelize function activations within the VM. `AUTO` uses as many processes as CPUs the VM has | |vm | extra_apt_packages | [] | no | Extra Debian/Ubuntu packages during Lithops setup on the VM (YAML list or space-separated string) | |vm | extra_python_packages | [] | no | Extra pip packages after Lithops on the VM (YAML list or space-separated string) | diff --git a/docs/source/data_processing.rst b/docs/source/data_processing.rst index b03f803e0..c1469f573 100644 --- a/docs/source/data_processing.rst +++ b/docs/source/data_processing.rst @@ -15,7 +15,7 @@ Lithops includes an integrated **data partitioner** that allows you to automatic Partitioning is supported directly within the :meth:`map()` and :meth:`map_reduce()` APIs and can be controlled via: - **`obj_chunk_size`**: The size (in bytes) of each chunk to split the object into. -- **`obj_chunk_number`**: The total number of chunks to split each object into. +- **`obj_chunk_number`**: The total number of chunks to split each object into. An object with fewer bytes than that is split into fewer chunks, none of them empty (with ``obj_newline`` set, each chunk holds at least one byte more than the newline). Currently, the partitioner supports **text-based files** where rows are separated by newline characters (`\n`), such as `.txt` and `.csv`. Support for additional data types is planned in future releases. @@ -141,7 +141,7 @@ Processing Data from Public URLs Lithops also supports processing data directly from **public URLs**. The input can be a single URL or a list of URLs. -If you set ``obj_chunk_size`` or ``obj_chunk_number``, Lithops activates its internal partitioner to split each file into smaller chunks—**provided that the remote server supports HTTP range requests**. If range requests are not supported, each URL is processed as a single object. +If you set ``obj_chunk_size`` or ``obj_chunk_number``, Lithops activates its internal partitioner to split each file into smaller chunks—**provided that the remote server supports HTTP range requests** (its ``Accept-Ranges`` header is ``bytes``). If range requests are not supported, each URL is processed as a single object. Each URL is sized with a ``HEAD`` request that follows redirects; a URL that answers with an error status raises an error instead of being processed. As with other backends, the special ``obj`` parameter gives you access to metadata and the content of the chunk being processed. @@ -196,7 +196,7 @@ Lithops can also process files stored on the local filesystem. The input can be: - A directory path - A list of directory paths -As in other cases, if you set ``obj_chunk_size`` or ``obj_chunk_number``, the file(s) will be split into chunks and processed in parallel. If not set, each file is processed as a single object. +As in other cases, if you set ``obj_chunk_size`` or ``obj_chunk_number``, the file(s) will be split into chunks and processed in parallel. If not set, each file is processed as a single object. A path that does not exist raises ``FileNotFoundError``, and directories that contain no files raise an error, as an empty bucket or prefix does. The ``obj`` parameter again exposes the metadata and content of the chunk. diff --git a/lithops/config.py b/lithops/config.py index 344f4fa41..ceefb48d5 100644 --- a/lithops/config.py +++ b/lithops/config.py @@ -179,7 +179,7 @@ def _resolve_mode_and_backend(config_data): mode = lithops_cfg.get('mode') if mode and not backend: - if mode in config_data and 'backend' in config_data[mode]: + if 'backend' in (config_data.get(mode) or {}): lithops_cfg['backend'] = config_data[mode]['backend'] else: lithops_cfg['backend'] = get_default_backend(mode) diff --git a/lithops/invokers.py b/lithops/invokers.py index e6691b235..68cee3701 100644 --- a/lithops/invokers.py +++ b/lithops/invokers.py @@ -450,10 +450,11 @@ def _async_invoker_loop(self, inv_id): def _start_async_invokers(self): """Starts the invoker process responsible to spawn - pending calls in background. + pending calls in background. They start with no token: the + max_workers free workers are invoked directly, and every later + one waits for a token handed back by a finished worker """ for inv_id in range(self.ASYNC_INVOKERS): - self.job_monitor.token_bucket_q.put('#') invoker = threading.Thread( target=self._async_invoker_loop, args=(inv_id,) ) diff --git a/lithops/job/partitioner.py b/lithops/job/partitioner.py index 632f12c8f..f4cb26893 100644 --- a/lithops/job/partitioner.py +++ b/lithops/job/partitioner.py @@ -17,6 +17,7 @@ import os import re +import errno import posixpath import logging import requests @@ -109,42 +110,40 @@ def _log_chunk_settings(chunk_size: Optional[int], chunk_number: Optional[int]) logger.debug('Chunk size and chunk number not set') -def _chunk_from_number(obj_size: int, chunk_number: int) -> int: - chunk_rest = obj_size % chunk_number - return (obj_size // chunk_number) + round((chunk_rest / chunk_number) + 0.5) - - -def _sized_object_chunk( - obj_size: Optional[int], +def _chunk_sizes( + obj_size: int, + chunk_size: Optional[int], chunk_number: Optional[int], - chunk_size: Optional[int] -) -> Optional[int]: + min_chunk: int +) -> List[int]: """ - Resolves the chunk size to use for one object. A requested chunk number - wins over a requested chunk size, and an unset one means a single chunk + The size of every chunk of one object. A requested chunk number wins over + a requested chunk size, and an unset one means a single chunk. An object + of fewer than chunk_number * min_chunk bytes gets fewer chunks, none of + them under min_chunk bytes """ - if chunk_number and obj_size: - return _chunk_from_number(obj_size, chunk_number) - if chunk_size and obj_size: - return chunk_size - if obj_size: - return obj_size - return None + if chunk_number: + chunk_number = max(1, min(chunk_number, obj_size // min_chunk)) + bounds = [i * obj_size // chunk_number for i in range(chunk_number + 1)] + return [end - start for start, end in zip(bounds, bounds[1:])] + chunk_size = max(chunk_size or obj_size, min_chunk) + return [chunk_size] * (obj_size // chunk_size + (obj_size % chunk_size > 0)) def _build_partitions( entry: Entry, make_obj: Callable[[], Any], obj_size: Optional[int], - obj_chunk_size: Optional[int], + chunk_size: Optional[int], + chunk_number: Optional[int], obj_newline: Optional[str], label: str ) -> Partitions: """ - Splits one object into as many partitions as its chunk size calls for, - each one a copy of the entry carrying its own byte range + Splits one object into as many partitions as its chunk size or number + calls for, each one a copy of the entry carrying its own byte range """ - if not obj_size or not obj_chunk_size: + if not obj_size: return [], 0 obj_partitions = [] @@ -152,14 +151,13 @@ def _build_partitions( # Every chunk but the first one starts as many bytes early as the newline # has. Longer than the newline, only the first chunk starts at byte 0 newline_len = len(obj_newline.encode()) if obj_newline is not None else 0 - obj_chunk_size = max(obj_chunk_size, newline_len + 1) + chunk_sizes = _chunk_sizes(obj_size, chunk_size, chunk_number, newline_len + 1) - parts = obj_size // obj_chunk_size + (obj_size % obj_chunk_size > 0) logger.debug( - f'Creating {parts} partitions from {label} ({sizeof_fmt(obj_size)})' + f'Creating {len(chunk_sizes)} partitions from {label} ({sizeof_fmt(obj_size)})' ) - while size < obj_size: + for obj_chunk_size in chunk_sizes: if obj_size <= obj_chunk_size: # A single partition reads the whole object, no range needed brange = None @@ -228,24 +226,32 @@ def _split_objects_from_urls( def _split(entry): obj_size = None object_url = entry['obj'] - metadata = requests.head(object_url) + # The worker reads the object with a GET, which follows redirects + metadata = requests.head(object_url, allow_redirects=True) + if not 200 <= metadata.status_code < 300: + raise requests.HTTPError( + f'HEAD {object_url} returned HTTP {metadata.status_code}, ' + 'so its size is unknown', response=metadata + ) if 'content-length' in metadata.headers: obj_size = int(metadata.headers['content-length']) - obj_chunk_size = _sized_object_chunk(obj_size, chunk_number, chunk_size) + obj_chunk_size, obj_chunk_number = chunk_size, chunk_number if obj_size is None: # Size unknown, so the object is a single partition of unset size - obj_chunk_size = obj_size = 1 + obj_size = 1 - if 'accept-ranges' not in metadata.headers: - obj_chunk_size = obj_size + if 'bytes' not in metadata.headers.get('accept-ranges', '').lower(): + # Without range requests, every partition would read the whole object + obj_chunk_size = obj_chunk_number = None return _build_partitions( entry, lambda: CloudObjectUrl(object_url), obj_size, obj_chunk_size, + obj_chunk_number, obj_newline, f'url {object_url}' ) @@ -256,12 +262,15 @@ def _split(entry): def _expand_paths(map_func_args_list: List[Entry]) -> List[Entry]: """ Replaces every directory entry with one entry per file it contains, - dropping duplicates and anything that is not a file + dropping duplicates and anything that is not a file. A path that does + not exist raises FileNotFoundError """ files = set() expanded = [] for elem in map_func_args_list: + if not os.path.exists(elem['obj']): + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), elem['obj']) if os.path.isdir(elem['obj']): path = elem['obj'] for filename in os.listdir(path): @@ -295,17 +304,21 @@ def _split_objects_from_paths( def _split(entry): path = entry['obj'] obj_size = int(os.stat(path).st_size) - obj_chunk_size = _sized_object_chunk(obj_size, chunk_number, chunk_size) return _build_partitions( entry, lambda: CloudObjectLocal(path), obj_size, - obj_chunk_size, + chunk_size, + chunk_number, obj_newline, f'path {path}' ) - return _collect_partitions(_split, _expand_paths(map_func_args_list)) + files = _expand_paths(map_func_args_list) + if not files: + raise Exception('No objects found') + + return _collect_partitions(_split, files) def _glob_list_prefix(pattern: str) -> str: @@ -445,15 +458,12 @@ def _split_objects_from_object_storage( continue total_objects += 1 - obj_size = dobj['Size'] - obj_chunk_size = _sized_object_chunk( - obj_size, chunk_number, chunk_size - ) obj_partitions, nparts = _build_partitions( {'obj': f'{sb}://{bucket}/{key}', **params}, lambda: CloudObject(sb, bucket, key), - obj_size, - obj_chunk_size, + dobj['Size'], + chunk_size, + chunk_number, obj_newline, f'object {key}' ) diff --git a/lithops/scripts/cli.py b/lithops/scripts/cli.py index 1db2c4df0..8933923fb 100644 --- a/lithops/scripts/cli.py +++ b/lithops/scripts/cli.py @@ -829,6 +829,7 @@ def update(name, config, backend, storage, debug): continue runtime_key = handler.get_runtime_key(rt[0], rt[1], rt[2]) runtime_meta = handler.deploy_runtime(rt[0], rt[1], runtime_timeout) + runtime_meta['runtime_timeout'] = runtime_timeout internal_storage.put_runtime_meta(runtime_key, runtime_meta) logger.info('Runtime updated') diff --git a/lithops/serverless/backends/aliyun_fc/aliyun_fc.py b/lithops/serverless/backends/aliyun_fc/aliyun_fc.py index 8d3a109d6..e4158697b 100644 --- a/lithops/serverless/backends/aliyun_fc/aliyun_fc.py +++ b/lithops/serverless/backends/aliyun_fc/aliyun_fc.py @@ -539,7 +539,6 @@ def _generate_runtime_meta(self, function_name, max_attempts, retry_sleep): headers = fc_models.InvokeFunctionHeaders( x_fc_invocation_type='Sync', ) - request = fc_models.InvokeFunctionRequest(body=io.BytesIO(payload_bytes)) last_error = None runtime_meta = None @@ -552,6 +551,8 @@ def _generate_runtime_meta(self, function_name, max_attempts, retry_sleep): pass try: + # The body stream is consumed by each attempt + request = fc_models.InvokeFunctionRequest(body=io.BytesIO(payload_bytes)) res = self.fc_client.invoke_function_with_options( function_name, request, headers, FC_INVOKE_RUNTIME ) diff --git a/lithops/serverless/backends/aws_batch/aws_batch.py b/lithops/serverless/backends/aws_batch/aws_batch.py index 527822803..500aacb1a 100644 --- a/lithops/serverless/backends/aws_batch/aws_batch.py +++ b/lithops/serverless/backends/aws_batch/aws_batch.py @@ -101,15 +101,20 @@ def _get_full_image_name(self, runtime_name): def _format_jobdef_name(self, runtime_name, runtime_memory, version=__version__): fmt_runtime_name = runtime_name.replace('/', '--').replace(':', '--') - package = self.package.replace(__version__.replace(".", "-"), version.replace(".", "-")) + package = self.package.replace( + f'lithops_v{__version__.replace(".", "")}_', f'lithops_v{version.replace(".", "")}_', 1 + ) return f'{package}--{self.env_type}--{fmt_runtime_name}--{runtime_memory}mb' def _unformat_jobdef_name(self, jobdef_name): - # Default jobdef name is "lithops_v2-7-2_WU5O--FARGATE_SPOT--batch-default-runtime-v310--latest--1024mb" + # Default jobdef name is "lithops_v381dev0_wu5o--FARGATE_SPOT--batch-default-runtime-v310--latest--1024mb" prefix, env_type, runtime = jobdef_name.split('--', 2) - version = prefix.replace('lithops_v', '').split('_')[0].replace('-', '.') + version = prefix.replace('lithops_v', '', 1).split('_')[0] + # Job definition names cannot contain dots, so only the current version can be restored + if version == __version__.replace('.', ''): + version = __version__ runtime_name, memory = runtime.rsplit("--", 1) - return runtime_name.replace('--', ':'), memory.replace('mb', ''), version + return runtime_name.replace('--', ':'), int(memory.replace('mb', '')), version def _build_default_runtime(self, runtime_name): # Build default runtime using local docker @@ -258,6 +263,14 @@ def _create_job_def(self, runtime_name, runtime_memory): platform_capabilities = ['EC2'] elif self.env_type in {'FARGATE', 'FARGATE_SPOT'}: platform_capabilities = ['FARGATE'] + runtime_cpu = self.aws_batch_config['runtime_cpu'] + mem_keys = batch_config.AVAILABLE_CPU_MEM_FARGATE.get(runtime_cpu, []) + if runtime_memory not in mem_keys: + raise Exception( + f"'{runtime_memory}' runtime memory is not valid for {runtime_cpu} " + f"vCPU and the {self.env_type} environment (for {runtime_cpu}vCPU " + f"choose one of {', '.join(map(str, mem_keys))})" + ) else: raise Exception(f'Unknown env type {self.env_type}') @@ -276,7 +289,7 @@ def _create_job_def(self, runtime_name, runtime_memory): }, { 'type': 'MEMORY', - 'value': str(self.aws_batch_config['runtime_memory']) + 'value': str(runtime_memory) } ], } @@ -303,17 +316,23 @@ def _create_job_def(self, runtime_name, runtime_memory): logger.debug('Using existing Job Definition {}'.format(job_def_name)) def _get_job_def(self, jd_name=None): - res = self.batch_client.describe_job_definitions(status='ACTIVE') - - if res['ResponseMetadata']['HTTPStatusCode'] != 200: - raise Exception(res) + all_job_defs = [] + kwargs = {'status': 'ACTIVE'} + while True: + res = self.batch_client.describe_job_definitions(**kwargs) + if res['ResponseMetadata']['HTTPStatusCode'] != 200: + raise Exception(res) + all_job_defs.extend(res['jobDefinitions']) + if not res.get('nextToken'): + break + kwargs['nextToken'] = res['nextToken'] if jd_name is None: - job_defs = [jd for jd in res['jobDefinitions'] + job_defs = [jd for jd in all_job_defs if self.package in jd['jobDefinitionName']] return job_defs - job_defs = [jd for jd in res['jobDefinitions'] if jd['jobDefinitionName'] == jd_name] + job_defs = [jd for jd in all_job_defs if jd['jobDefinitionName'] == jd_name] if len(job_defs) == 0: return None if len(job_defs) == 1: @@ -428,6 +447,9 @@ def deploy_runtime(self, runtime_name, memory, timeout=900): def delete_runtime(self, runtime_name, runtime_memory, version=__version__): jobdef_name = self._format_jobdef_name(runtime_name, runtime_memory, version) job_def = self._get_job_def(jobdef_name) + if job_def is None: + logger.info(f'Job definition {jobdef_name} does not exist') + return logger.info('Deleting job definition with ARN {}'.format(job_def['jobDefinitionArn'])) res = self.batch_client.deregister_job_definition(jobDefinition=job_def['jobDefinitionArn']) @@ -527,8 +549,8 @@ def invoke(self, runtime_name, runtime_memory, payload): executor_id = payload['executor_id'] job_id = payload['job_id'] total_calls = payload['total_calls'] - max_workers = payload['max_workers'] - chunksize = payload['chunksize'] + max_workers = int(payload['max_workers']) + chunksize = int(payload['chunksize']) # Make sure only max_workers are started total_workers = total_calls // chunksize + (total_calls % chunksize > 0) diff --git a/lithops/serverless/backends/aws_batch/config.py b/lithops/serverless/backends/aws_batch/config.py index dc3a21f13..ce1112302 100644 --- a/lithops/serverless/backends/aws_batch/config.py +++ b/lithops/serverless/backends/aws_batch/config.py @@ -137,8 +137,9 @@ def load_config(config_data): "['m5.large','c5.xlarge'], ['m5'], or ['optimal']" ) - config_data['aws_batch']['max_workers'] = config_data['aws_batch']['env_max_cpus'] \ - // config_data['aws_batch']['runtime_cpu'] + config_data['aws_batch']['max_workers'] = int( + config_data['aws_batch']['env_max_cpus'] // config_data['aws_batch']['runtime_cpu'] + ) assert isinstance(config_data['aws_batch']['assign_public_ip'], bool) diff --git a/lithops/serverless/backends/aws_lambda/aws_lambda.py b/lithops/serverless/backends/aws_lambda/aws_lambda.py index 326360524..e8e9b0252 100644 --- a/lithops/serverless/backends/aws_lambda/aws_lambda.py +++ b/lithops/serverless/backends/aws_lambda/aws_lambda.py @@ -38,6 +38,8 @@ LITHOPS_FUNCTION_ZIP = 'lithops_lambda.zip' BUILD_LAYER_FUNCTION_ZIP = 'build_layer.zip' +INVOKE_RETRIES = 5 +INVOKE_RETRY_SLEEP = 1 # seconds, multiplied by the attempt number class AWSLambdaBackend: @@ -147,8 +149,9 @@ def _create_handler_bin(remove=True): def _wait_for_function_deployed(self, func_name): """ - Helper function which waits for the lambda to be deployed (state is 'Active'). - Raises exception if waiting times out or if state is 'Failed' or 'Inactive' + Helper function which waits for the lambda to be deployed (state is 'Active') + and for its last update to finish. + Raises exception if waiting times out, if state is 'Failed' or 'Inactive' or if the last update failed """ retries, sleep_seconds = (15, 25) if 'vpc' in self.lambda_config else (30, 5) logger.debug( @@ -158,7 +161,8 @@ def _wait_for_function_deployed(self, func_name): while retries > 0: res = self.lambda_client.get_function(FunctionName=func_name) state = res['Configuration']['State'] - if state == 'Pending': + update_status = res['Configuration'].get('LastUpdateStatus') + if state == 'Pending' or (state == 'Active' and update_status == 'InProgress'): logger.debug( f'"{func_name}" function is being deployed... (status: {state})' ) @@ -172,6 +176,10 @@ def _wait_for_function_deployed(self, func_name): raise Exception( f'"{func_name}" function not deployed (state is "{state}"): {res}' ) + elif state == 'Active' and update_status == 'Failed': + raise Exception( + f'"{func_name}" function not updated (last update status is "{update_status}"): {res}' + ) elif state == 'Active': break @@ -291,7 +299,7 @@ def _delete_layer(self, layer_name): versions.extend([layer['Version'] for layer in response['LayerVersions']]) while 'NextMarker' in response: - response = self.lambda_client.list_layer_versions(Marker=response['NextMarker']) + response = self.lambda_client.list_layer_versions(LayerName=layer_name, Marker=response['NextMarker']) versions.extend([layer['Version'] for layer in response['LayerVersions']]) for version in versions: @@ -308,9 +316,9 @@ def _list_layers(self): @return: list of Lithops layer ARNs """ logger.debug('Listing lambda layers') - response = self.lambda_client.list_layers() - - layers = response['Layers'] if 'Layers' in response else [] + layers = [] + for page in self.lambda_client.get_paginator('list_layers').paginate(): + layers.extend(page.get('Layers', [])) logger.debug('Listed {} layers'.format(len(layers))) lithops_layers = [] for layer in layers: @@ -324,9 +332,15 @@ def _delete_function(self, function_name): @param function_name: function name to delete """ logger.info(f'Deleting function: {function_name}') - response = self.lambda_client.delete_function( - FunctionName=function_name - ) + try: + response = self.lambda_client.delete_function( + FunctionName=function_name + ) + except botocore.exceptions.ClientError as err: + if err.response['Error']['Code'] != 'ResourceNotFoundException': + raise + logger.debug(f'OK --> Function {function_name} does not exist') + return if response['ResponseMetadata']['HTTPStatusCode'] == 204: logger.debug('OK --> Deleted function {}'.format(function_name)) @@ -402,7 +416,33 @@ def _deploy_default_runtime(self, runtime_name, memory, timeout): layer_arn = self._create_layer(runtime_name) code = self._create_handler_bin() - env_vars = {t['name']: t['value'] for t in self.lambda_config['env_vars']} + + function_config = dict( + Runtime=config.AVAILABLE_PY_RUNTIMES[utils.CURRENT_PY_VERSION], + Role=self.role_arn, + Handler='entry_point.lambda_handler', + Description=f'Lithops worker for v{__version__} and Python v{utils.CURRENT_PY_VERSION}', + Timeout=timeout, + MemorySize=memory, + Layers=[layer_arn], + VpcConfig={ + 'SubnetIds': self.lambda_config['vpc']['subnets'], + 'SecurityGroupIds': self.lambda_config['vpc']['security_groups'] + }, + FileSystemConfigs=[ + {'Arn': efs_conf['access_point'], + 'LocalMountPath': efs_conf['mount_path']} + for efs_conf in self.lambda_config['efs'] + ], + EphemeralStorage={ + 'Size': self.lambda_config['ephemeral_storage'] + }, + Environment={ + 'Variables': self.lambda_config['env_vars'] + } + ) + + tags = self._function_tags(runtime_name) logger.debug( f'Creating Lambda function {function_name} ' @@ -411,35 +451,11 @@ def _deploy_default_runtime(self, runtime_name, memory, timeout): try: response = self.lambda_client.create_function( FunctionName=function_name, - Runtime=config.AVAILABLE_PY_RUNTIMES[utils.CURRENT_PY_VERSION], - Role=self.role_arn, - Handler='entry_point.lambda_handler', Code={ 'ZipFile': code }, - Description=f'Lithops worker for v{__version__} and Python v{utils.CURRENT_PY_VERSION}', - Timeout=timeout, - MemorySize=memory, - Layers=[layer_arn], - VpcConfig={ - 'SubnetIds': self.lambda_config['vpc']['subnets'], - 'SecurityGroupIds': self.lambda_config['vpc']['security_groups'] - }, - FileSystemConfigs=[ - {'Arn': efs_conf['access_point'], - 'LocalMountPath': efs_conf['mount_path']} - for efs_conf in self.lambda_config['efs'] - ], - Tags={ - 'runtime_name': runtime_name, - 'lithops_version': __version__ - }, - EphemeralStorage={ - 'Size': self.lambda_config['ephemeral_storage'] - }, - Environment={ - 'Variables': env_vars - } + Tags=tags, + **function_config ) if response['ResponseMetadata']['HTTPStatusCode'] not in (200, 201): @@ -453,16 +469,37 @@ def _deploy_default_runtime(self, runtime_name, memory, timeout): except Exception as e: if 'ResourceConflictException' in str(e): - logger.debug( - f'Lambda function {function_name} already exists, ' - 'waiting until it is active' - ) + self._update_function(function_name, {'ZipFile': code}, function_config, tags) else: raise e self._wait_for_function_deployed(function_name) logger.debug(f'OK --> Created lambda function {function_name}') + def _function_tags(self, runtime_name): + return { + 'runtime_name': runtime_name, + 'lithops_version': __version__, + **self.lambda_config['user_tags'], + } + + def _update_function(self, function_name, code, function_config, tags): + """ + Updates the code, the configuration and the tags of an already deployed function. + Tags are added or overwritten, never removed, as other tools may have set them + @param function_name: function name to update + @param code: update_function_code() arguments + @param function_config: update_function_configuration() arguments + @param tags: tags the function must have + """ + logger.debug(f'Lambda function {function_name} already exists, updating its code and configuration') + # Lambda rejects an update while the function is still being created or updated + self._wait_for_function_deployed(function_name) + self.lambda_client.update_function_code(FunctionName=function_name, **code) + self._wait_for_function_deployed(function_name) + response = self.lambda_client.update_function_configuration(FunctionName=function_name, **function_config) + self.lambda_client.tag_resource(Resource=response['FunctionArn'], Tags=tags) + def _deploy_container_runtime(self, runtime_name, memory, timeout): """ Deploy a runtime based on a container @@ -490,40 +527,39 @@ def _deploy_container_runtime(self, runtime_name, memory, timeout): registry = f'{self.account_id}.dkr.ecr.{self.region}.amazonaws.com' image_uri = f'{registry}/{repo_name}@{image_digest}' - env_vars = {t['name']: t['value'] for t in self.lambda_config['env_vars']} + function_config = dict( + Role=self.role_arn, + Description=f'Lithops worker for v{__version__} and Python v{utils.CURRENT_PY_VERSION}', + Timeout=timeout, + MemorySize=memory, + VpcConfig={ + 'SubnetIds': self.lambda_config['vpc']['subnets'], + 'SecurityGroupIds': self.lambda_config['vpc']['security_groups'] + }, + FileSystemConfigs=[ + {'Arn': efs_conf['access_point'], + 'LocalMountPath': efs_conf['mount_path']} + for efs_conf in self.lambda_config['efs'] + ], + EphemeralStorage={ + 'Size': self.lambda_config['ephemeral_storage'] + }, + Environment={ + 'Variables': self.lambda_config['env_vars'] + } + ) + tags = self._function_tags(runtime_name) try: response = self.lambda_client.create_function( FunctionName=function_name, - Role=self.role_arn, Code={ 'ImageUri': image_uri }, PackageType='Image', - Description=f'Lithops worker for v{__version__} and Python v{utils.CURRENT_PY_VERSION}', - Timeout=timeout, - MemorySize=memory, - VpcConfig={ - 'SubnetIds': self.lambda_config['vpc']['subnets'], - 'SecurityGroupIds': self.lambda_config['vpc']['security_groups'] - }, - FileSystemConfigs=[ - {'Arn': efs_conf['access_point'], - 'LocalMountPath': efs_conf['mount_path']} - for efs_conf in self.lambda_config['efs'] - ], - Tags={ - 'runtime_name': runtime_name, - 'lithops_version': __version__, - **self.lambda_config.get('user_tags'), - }, + Tags=tags, Architectures=[self.lambda_config['architecture']], - EphemeralStorage={ - 'Size': self.lambda_config['ephemeral_storage'] - }, - Environment={ - 'Variables': env_vars - } + **function_config ) if response['ResponseMetadata']['HTTPStatusCode'] not in (200, 201): @@ -531,7 +567,8 @@ def _deploy_container_runtime(self, runtime_name, memory, timeout): except Exception as e: if 'ResourceConflictException' in str(e): - pass + code = {'ImageUri': image_uri, 'Architectures': [self.lambda_config['architecture']]} + self._update_function(function_name, code, function_config, tags) else: raise e @@ -584,11 +621,8 @@ def delete_runtimes(response): if function['FunctionName'].startswith(prefix): self._delete_function(function['FunctionName']) - response = self.lambda_client.list_functions(FunctionVersion='ALL') - delete_runtimes(response) - while 'NextMarker' in response: - response = self.lambda_client.list_functions(Marker=response['NextMarker']) - delete_runtimes(response) + for page in self.lambda_client.get_paginator('list_functions').paginate(): + delete_runtimes(page) layers = self._list_layers() for layer_name, _ in layers: @@ -617,11 +651,8 @@ def get_runtimes(response): version = tags_response['lithops_version'] runtimes.append((rt_name, rt_memory, version, fn_name)) - response = self.lambda_client.list_functions(FunctionVersion='ALL') - get_runtimes(response) - while 'NextMarker' in response: - response = self.lambda_client.list_functions(Marker=response['NextMarker']) - get_runtimes(response) + for page in self.lambda_client.get_paginator('list_functions').paginate(): + get_runtimes(page) if runtime_name != 'all': runtimes = [tup for tup in runtimes if runtime_name == tup[0]] @@ -641,16 +672,21 @@ def invoke(self, runtime_name, runtime_memory, payload): headers = {'Host': self.host, 'X-Amz-Invocation-Type': 'Event', 'User-Agent': self.user_agent} url = f'https://{self.host}/2015-03-31/functions/{function_name}/invocations' - request = AWSRequest(method="POST", url=url, data=json.dumps(payload, default=str), headers=headers) - SigV4Auth(self.credentials, "lambda", self.region).add_auth(request) + data = json.dumps(payload, default=str) - invoked = False - while not invoked: + for attempt in range(1, INVOKE_RETRIES + 1): + request = AWSRequest(method="POST", url=url, data=data, headers=headers) + SigV4Auth(self.credentials, "lambda", self.region).add_auth(request) try: r = self.session.send(request.prepare()) - invoked = True - except Exception: - pass + break + # Only errors raised before the request reached Lambda: resending after it was + # accepted would run the calls twice + except (botocore.exceptions.EndpointConnectionError, botocore.exceptions.ConnectTimeoutError) as e: + if attempt == INVOKE_RETRIES: + raise + logger.debug(f'Invocation of {function_name} failed ({e}), retrying ({attempt}/{INVOKE_RETRIES})') + time.sleep(INVOKE_RETRY_SLEEP * attempt) if r.status_code == 202: return r.headers['x-amzn-RequestId'] diff --git a/lithops/serverless/backends/aws_lambda/config.py b/lithops/serverless/backends/aws_lambda/config.py index a1c9cfcb5..a34f7eecb 100644 --- a/lithops/serverless/backends/aws_lambda/config.py +++ b/lithops/serverless/backends/aws_lambda/config.py @@ -65,6 +65,21 @@ RUNTIME_TMP_SZ_MAX = 10240 +def _load_name_value_pairs(value, key): + """ + Returns a {NAME: VALUE} dict from either a {NAME: VALUE} dict or a list of {name: ..., value: ...} pairs + """ + if not value: + return {} + if isinstance(value, dict): + return {str(k): str(v) for k, v in value.items()} + if isinstance(value, list) and all(isinstance(p, dict) and {'name', 'value'} <= set(p) for p in value): + return {str(p['name']): str(p['value']) for p in value} + raise Exception( + f"'aws_lambda/{key}' must be a {{NAME: VALUE}} mapping or a list of {{name: ..., value: ...}} pairs" + ) + + def load_config(config_data): if not config_data['aws_lambda']: @@ -129,6 +144,9 @@ def load_config(config_data): or config_data['aws_lambda']['ephemeral_storage'] > RUNTIME_TMP_SZ_MAX: raise Exception(f'Ephemeral storage value must be between {RUNTIME_TMP_SZ_MIN} and {RUNTIME_TMP_SZ_MAX}') + for key in ('env_vars', 'user_tags'): + config_data['aws_lambda'][key] = _load_name_value_pairs(config_data['aws_lambda'][key], key) + if 'region_name' in config_data['aws_lambda']: config_data['aws_lambda']['region'] = config_data['aws_lambda'].pop('region_name') diff --git a/lithops/serverless/backends/azure_containers/azure_containers.py b/lithops/serverless/backends/azure_containers/azure_containers.py index bf729a600..937504825 100644 --- a/lithops/serverless/backends/azure_containers/azure_containers.py +++ b/lithops/serverless/backends/azure_containers/azure_containers.py @@ -470,9 +470,9 @@ def list_runtimes(self, runtime_name='all'): if containerapp['Tags'] and 'type' in containerapp['Tags'] \ and containerapp['Tags']['type'] == 'lithops-runtime': name = containerapp['Tags']['runtime_name'] - memory = containerapp['Tags']['runtime_memory'] + memory = int(containerapp['Tags']['runtime_memory']) version = containerapp['Tags']['lithops_version'] - if runtime_name == containerapp['Name'] or runtime_name == 'all': + if runtime_name == name or runtime_name == 'all': runtimes.append((name, memory, version, containerapp['Name'])) return runtimes diff --git a/lithops/serverless/backends/azure_functions/azure_functions.py b/lithops/serverless/backends/azure_functions/azure_functions.py index 532865237..7f628bcf6 100644 --- a/lithops/serverless/backends/azure_functions/azure_functions.py +++ b/lithops/serverless/backends/azure_functions/azure_functions.py @@ -181,26 +181,31 @@ def _publish_function(self, function_name, build_dir): time.sleep(10) - def _get_function_identity(self, runtime_name, version=__version__): + def _get_function_identity(self, runtime_name, memory, version=__version__): """ Returns a stable hash for a Lithops runtime on Azure Functions. """ - name = f'{self.storage_account_name}-{runtime_name}-{version}-{self.trigger}' + instance_memory = config.get_flex_instance_memory(int(memory)) + name = f'{self.storage_account_name}-{runtime_name}-{version}-{self.trigger}-{instance_memory}' return hashlib.sha1(name.encode("utf-8")).hexdigest()[:12] - def _format_function_name(self, runtime_name, version=__version__): + def _format_function_name(self, runtime_name, memory, version=__version__): """ Formats the function app name. Host ID collisions are avoided via AzureFunctionsWebHost__hostid. """ - name_hash = self._get_function_identity(runtime_name, version) + # The memory is only part of the hash: app names are limited to 60 characters + name_hash = self._get_function_identity(runtime_name, memory, version) return f'lithops-worker-{runtime_name}-{version.replace(".", "")}-{name_hash}' - def _format_host_id(self, runtime_name, version=__version__): + def _format_host_id(self, runtime_name, memory, version=__version__): """ Unique Functions host ID (must start with a letter, alphanumeric only). """ - return f'l{self._get_function_identity(runtime_name, version)}' + return f'l{self._get_function_identity(runtime_name, memory, version)}' + + def _format_build_dir(self, runtime_name, version=__version__): + return os.path.join(config.BUILD_DIR, f'lithops-worker-{runtime_name}-{version.replace(".", "")}') def _format_queue_name(self, function_name, q_type): """ @@ -253,9 +258,7 @@ def build_runtime(self, runtime_name, requirements_file, extra_args=[]): except Exception: pass - function_name = self._format_function_name(runtime_name) - - build_dir = os.path.join(config.BUILD_DIR, function_name) + build_dir = self._format_build_dir(runtime_name) os.makedirs(build_dir, exist_ok=True) action_dir = os.path.join(build_dir, config.ACTION_DIR) @@ -279,19 +282,6 @@ def build_runtime(self, runtime_name, requirements_file, extra_args=[]): with open(host_file, 'w') as hstf: hstf.write(config.HOST_FILE) - fn_file = os.path.join(action_dir, 'function.json') - if self.trigger == 'pub/sub': - with open(fn_file, 'w') as fnf: - in_q_name = self._format_queue_name(function_name, config.IN_QUEUE) - config.BINDINGS_QUEUE['bindings'][0]['queueName'] = in_q_name - out_q_name = self._format_queue_name(function_name, config.OUT_QUEUE) - config.BINDINGS_QUEUE['bindings'][1]['queueName'] = out_q_name - fnf.write(json.dumps(config.BINDINGS_QUEUE)) - - elif self.trigger == 'https': - with open(fn_file, 'w') as fnf: - fnf.write(json.dumps(config.BINDINGS_HTTP)) - entry_point = os.path.join(os.path.dirname(__file__), 'entry_point.py') main_file = os.path.join(action_dir, '__init__.py') shutil.copy(entry_point, main_file) @@ -309,12 +299,29 @@ def build_runtime(self, runtime_name, requirements_file, extra_args=[]): logger.debug(f'Runtime {runtime_name} built successfully') + def _write_function_bindings(self, build_dir, function_name): + """ + Writes the function.json of the app, the queue names depend on the app name + """ + fn_file = os.path.join(build_dir, config.ACTION_DIR, 'function.json') + if self.trigger == 'pub/sub': + with open(fn_file, 'w') as fnf: + in_q_name = self._format_queue_name(function_name, config.IN_QUEUE) + config.BINDINGS_QUEUE['bindings'][0]['queueName'] = in_q_name + out_q_name = self._format_queue_name(function_name, config.OUT_QUEUE) + config.BINDINGS_QUEUE['bindings'][1]['queueName'] = out_q_name + fnf.write(json.dumps(config.BINDINGS_QUEUE)) + + elif self.trigger == 'https': + with open(fn_file, 'w') as fnf: + fnf.write(json.dumps(config.BINDINGS_HTTP)) + def _create_function(self, runtime_name, memory, timeout): """ Create and publish an Azure Functions """ logger.info(f'Creating Azure Function from runtime {runtime_name}') - function_name = self._format_function_name(runtime_name) + function_name = self._format_function_name(runtime_name, memory) instance_memory = config.get_flex_instance_memory(memory) if instance_memory != memory: @@ -331,19 +338,21 @@ def _create_function(self, runtime_name, memory, timeout): f'--runtime-version {utils.CURRENT_PY_VERSION} ' f'--functions-version {self.functions_version} ' f'--instance-memory {instance_memory} ' - f'--tags type=lithops-runtime lithops_version={__version__} runtime_name={runtime_name}') + f'--tags type=lithops-runtime lithops_version={__version__} runtime_name={runtime_name} ' + f'runtime_memory={instance_memory}') if not self._function_app_exists(function_name): self._run_az_command(cmd) else: logger.debug(f'Function app {function_name} already exists, skipping create') - host_id = self._format_host_id(runtime_name) + host_id = self._format_host_id(runtime_name, memory) cmd = (f'az functionapp config appsettings set --name {function_name} ' f'--resource-group {self.resource_group} ' f'--settings AzureFunctionsWebHost__hostid={host_id}') self._run_az_command(cmd) - build_dir = os.path.join(config.BUILD_DIR, function_name) + build_dir = self._format_build_dir(runtime_name) + self._write_function_bindings(build_dir, function_name) self._publish_function(function_name, build_dir) if self.trigger == 'pub/sub': @@ -360,7 +369,12 @@ def delete_runtime(self, runtime_name, memory, version=__version__, function_nam """ logger.info(f'Deleting runtime: {runtime_name} - {memory}MB') if function_name is None: - function_name = self._format_function_name(runtime_name, version) + function_name = self._format_function_name(runtime_name, memory, version) + # Apps deployed before the memory was part of the name only match through their tags + for rt_name, rt_memory, rt_version, wk_name in self.list_runtimes(runtime_name): + if rt_version == version and rt_memory == config.get_flex_instance_memory(int(memory)): + function_name = wk_name + break cmd = f'az functionapp delete --name {function_name} --resource-group {self.resource_group}' self._run_az_command(cmd) @@ -379,7 +393,7 @@ def invoke(self, runtime_name, memory=None, payload={}, return_result=False): """ Invoke function """ - function_name = self._format_function_name(runtime_name) + function_name = self._format_function_name(runtime_name, memory) if self.trigger == 'pub/sub': in_q_name = self._format_queue_name(function_name, config.IN_QUEUE) @@ -390,8 +404,15 @@ def invoke(self, runtime_name, memory=None, payload={}, return_result=False): if return_result: out_q_name = self._format_queue_name(function_name, config.OUT_QUEUE) out_queue = self.queue_service.get_queue_client(out_q_name) + timeout = self.af_config['runtime_timeout'] + deadline = time.time() + timeout msg = [] while not msg: + if time.time() > deadline: + raise Exception( + f'No response from function {function_name} in queue {out_q_name} ' + f'after {timeout} seconds. Review the function logs in the Azure Portal' + ) time.sleep(1) msg = out_queue.receive_message() out_queue.clear_messages() @@ -426,6 +447,8 @@ def invoke(self, runtime_name, memory=None, payload={}, return_result=False): if resp.status == 429: time.sleep(0.2) return None + if not 200 <= resp.status < 300: + raise Exception(f'Invocation error: ({resp.status}) {resp.reason} - {resp_text}') return resp_text @@ -435,7 +458,7 @@ def get_runtime_key(self, runtime_name, runtime_memory, version=__version__): Runtime keys are used to uniquely identify runtimes within the storage, in order to know which runtimes are installed and which not. """ - function_name = self._format_function_name(runtime_name, version) + function_name = self._format_function_name(runtime_name, runtime_memory, version) runtime_key = os.path.join(self.name, version, function_name) return runtime_key @@ -514,8 +537,8 @@ def list_runtimes(self, runtime_name='all'): and functionapp['Tags']['type'] == 'lithops-runtime': version = functionapp['Tags']['lithops_version'] name = functionapp['Tags']['runtime_name'] - memory = config.DEFAULT_CONFIG_KEYS['runtime_memory'] - if runtime_name == functionapp['Name'] or runtime_name == 'all': + memory = int(functionapp['Tags'].get('runtime_memory', config.DEFAULT_CONFIG_KEYS['runtime_memory'])) + if runtime_name == name or runtime_name == 'all': runtimes.append((name, memory, version, functionapp['Name'])) return runtimes diff --git a/lithops/serverless/backends/code_engine/code_engine.py b/lithops/serverless/backends/code_engine/code_engine.py index 02df47a23..f6c6a1945 100644 --- a/lithops/serverless/backends/code_engine/code_engine.py +++ b/lithops/serverless/backends/code_engine/code_engine.py @@ -465,7 +465,8 @@ def _wait_for_job_run(self, jobrun_name): Waits until a job run completes or raises if it fails """ logger.debug(f"Waiting for job run {jobrun_name}") - while True: + deadline = time.time() + config.JOB_RUN_TIMEOUT + while time.time() < deadline: try: job_run = self.ce_client.get_job_run(self.project_id, jobrun_name).get_result() except ApiException as e: @@ -485,6 +486,10 @@ def _wait_for_job_run(self, jobrun_name): ) time.sleep(config.JOB_RUN_POLL_INTERVAL) + raise Exception( + f"Timed out after {config.JOB_RUN_TIMEOUT}s waiting for job run {jobrun_name} to complete" + ) + @retry_on_except def _create_config_map(self, config_map_name, payload): """ @@ -504,6 +509,7 @@ def _create_config_map(self, config_map_name, payload): self.ce_client.replace_config_map( self.project_id, config_map_name, + if_match='*', data=data, ) else: @@ -687,7 +693,13 @@ def _create_job_definition(self, docker_image_name, runtime_memory, timeout=None if e.status_code != 404: raise e + deadline = time.time() + config.JOB_DEF_DELETE_TIMEOUT while self._job_def_exists(jobdef_name): + if time.time() >= deadline: + raise Exception( + f"Timed out after {config.JOB_DEF_DELETE_TIMEOUT}s waiting for job definition " + f"{jobdef_name} to be deleted" + ) time.sleep(1) kwargs = { @@ -791,7 +803,7 @@ def clear(self, job_keys=None): logger.info(f"Project {self.project_name} does not exist") return - for job_key in job_keys or self.jobs: + for job_key in list(job_keys or self.jobs): jobrun_name = f'lithops-{job_key.lower()}' self._delete_job_run(jobrun_name) self._delete_config_map(jobrun_name) diff --git a/lithops/serverless/backends/code_engine/config.py b/lithops/serverless/backends/code_engine/config.py index d238591e9..f9b65a90e 100644 --- a/lithops/serverless/backends/code_engine/config.py +++ b/lithops/serverless/backends/code_engine/config.py @@ -34,6 +34,8 @@ PYTHON_BIN = '/usr/local/bin/python' METADATA_JOBRUN_NAME = 'lithops-runtime-metadata' JOB_RUN_POLL_INTERVAL = 2 +JOB_RUN_TIMEOUT = 600 +JOB_DEF_DELETE_TIMEOUT = 300 PROJECT_POLL_INTERVAL = 10 PROJECT_READY_TIMEOUT = 600 diff --git a/lithops/serverless/backends/ibm_cf/ibm_cf.py b/lithops/serverless/backends/ibm_cf/ibm_cf.py index a9f1443c0..981f95116 100644 --- a/lithops/serverless/backends/ibm_cf/ibm_cf.py +++ b/lithops/serverless/backends/ibm_cf/ibm_cf.py @@ -16,7 +16,7 @@ import os import logging -from threading import Lock +from threading import RLock from lithops import utils from lithops.util.ibm_token_manager import IAMTokenManager @@ -28,7 +28,8 @@ from . import config logger = logging.getLogger(__name__) -cf_mutex = Lock() +# Reentrant: invoke() holds it while redeploying and deploy_runtime() refreshes the client under it +cf_mutex = RLock() class IBMCloudFunctionsBackend: @@ -95,20 +96,19 @@ def _refresh_ow_client(self): Refresh the OW client if necessary """ if not self.is_lithops_worker: - cf_mutex.acquire() - token, expiry_time = self.token_manager.get_token() + with cf_mutex: + token, expiry_time = self.token_manager.get_token() - if expiry_time != self.config['token_expiry_time']: - self.config['token'] = token - self.config['token_expiry_time'] = expiry_time + if expiry_time != self.config['token_expiry_time']: + self.config['token'] = token + self.config['token_expiry_time'] = expiry_time - self.cf_client = OpenWhiskClient( - endpoint=self.endpoint, - namespace=self.namespace_id, - auth='Bearer ' + token, - user_agent=self.user_agent - ) - cf_mutex.release() + self.cf_client = OpenWhiskClient( + endpoint=self.endpoint, + namespace=self.namespace_id, + auth='Bearer ' + token, + user_agent=self.user_agent + ) def _get_or_create_namespace(self, create=True): """ @@ -320,15 +320,14 @@ def invoke(self, docker_image_name, runtime_memory, payload): # Runtime not deployed if self.invoke_error is None: self.invoke_error = 404 - cf_mutex.acquire() - if self.invoke_error == 404: - logger.debug('Runtime not found') - self.deploy_runtime( - docker_image_name, runtime_memory, - self.config['runtime_timeout'] - ) - self.invoke_error = None - cf_mutex.release() + with cf_mutex: + if self.invoke_error == 404: + logger.debug('Runtime not found') + self.deploy_runtime( + docker_image_name, runtime_memory, + self.config['runtime_timeout'] + ) + self.invoke_error = None return self.invoke(docker_image_name, runtime_memory, payload) return activation_id @@ -377,7 +376,7 @@ def _generate_runtime_meta(self, docker_image_name, memory): if 'activationId' in runtime_meta: retry_invoke = True except Exception as e: - raise (f"Unable to extract metadata: {e}") + raise Exception(f"Unable to extract metadata: {e}") from e if not runtime_meta or 'preinstalls' not in runtime_meta: raise Exception(runtime_meta) diff --git a/lithops/serverless/backends/openwhisk/openwhisk.py b/lithops/serverless/backends/openwhisk/openwhisk.py index 6f01bdbca..01bae2a5c 100644 --- a/lithops/serverless/backends/openwhisk/openwhisk.py +++ b/lithops/serverless/backends/openwhisk/openwhisk.py @@ -213,6 +213,17 @@ def invoke(self, docker_image_name, runtime_memory, payload): activation_id = self.cf_client.invoke(self.package, action_name, payload, self.is_lithops_worker) + if activation_id == 401: + raise Exception( + f"Unauthorized invocation of {self.package}/{action_name} in namespace " + f"{self.namespace}: check the openwhisk api_key" + ) + elif activation_id == 404: + raise Exception( + f"Action {self.package}/{action_name} is not deployed in namespace {self.namespace}. " + "Run 'lithops clean -b openwhisk' and try again to redeploy the runtime" + ) + return activation_id def get_runtime_key(self, docker_image_name, runtime_memory, version=__version__): @@ -258,7 +269,7 @@ def _generate_runtime_meta(self, docker_image_name, memory): if 'activationId' in runtime_meta: retry_invoke = True except Exception as e: - raise (f"Unable to extract metadata: {e}") + raise Exception(f"Unable to extract metadata: {e}") from e if not runtime_meta or 'preinstalls' not in runtime_meta: raise Exception(runtime_meta) diff --git a/lithops/serverless/backends/oracle_f/oracle_f.py b/lithops/serverless/backends/oracle_f/oracle_f.py index 57921b1ce..f9b68b6c7 100644 --- a/lithops/serverless/backends/oracle_f/oracle_f.py +++ b/lithops/serverless/backends/oracle_f/oracle_f.py @@ -52,6 +52,7 @@ def __init__(self, oci_config, internal_storage): 'application_name', f'{config.APP_NAME}_{self.user[-8:-1].lower()}') self.signer = None + self.invoke_functions = {} self.cf_client = self._init_functions_mgmt_client() self.app_id = self._get_application_id(self.app_name) @@ -192,6 +193,12 @@ def pre_invoke(self, runtime_name, memory): """ Pre-invocation task. This is executed only once before the invocation """ + self._resolve_invoke_function(runtime_name, memory) + + def _resolve_invoke_function(self, runtime_name, memory): + """ + Looks up the function ID and invoke endpoint of a runtime and caches them + """ if not self.app_id: raise Exception(f'Application {self.app_name} does not exist') @@ -199,30 +206,33 @@ def pre_invoke(self, runtime_name, memory): function_name = self._format_function_name(image_name, memory) # Get the function ID - self.invoke_function_id = self._get_function_id(function_name) - if not self.invoke_function_id: - raise Exception("Function %s not found", function_name) + function_id = self._get_function_id(function_name) + if not function_id: + raise Exception(f'Function {function_name} not found') # Retrieve the function's information, including the invoke endpoint - fn_info = self.cf_client.get_function(self.invoke_function_id).data + fn_info = self.cf_client.get_function(function_id).data if not fn_info.lifecycle_state == 'ACTIVE': - raise Exception("Function %s is not yet active", function_name) + raise Exception(f'Function {function_name} is not yet active') - # Set the invoke endpoint - self.invoke_endpoint = fn_info.invoke_endpoint + self.invoke_functions[(runtime_name, memory)] = (function_id, fn_info.invoke_endpoint) + return function_id, fn_info.invoke_endpoint def invoke(self, runtime_name, memory, payload={}): """ Invoke function """ - # The pre_invoke() method is already called at this point + # Calls of a previous job can still be queued when pre_invoke() runs for the next one + if (runtime_name, memory) not in self.invoke_functions: + self._resolve_invoke_function(runtime_name, memory) + function_id, invoke_endpoint = self.invoke_functions[(runtime_name, memory)] # Prepare the Oracle Functions client with the invoke endpoint - fn_invoke_client = self._init_functions_invk_client(self.invoke_endpoint) + fn_invoke_client = self._init_functions_invk_client(invoke_endpoint) response = fn_invoke_client.invoke_function( - function_id=self.invoke_function_id, + function_id=function_id, invoke_function_body=json.dumps(payload, default=str), fn_invoke_type='detached' ) @@ -343,14 +353,14 @@ def _generate_runtime_meta(self, runtime_name, memory): image_name = self._format_image_name(runtime_name) logger.debug("Extracting runtime metadata from %s", image_name) - self.pre_invoke(runtime_name, memory) + function_id, invoke_endpoint = self._resolve_invoke_function(runtime_name, memory) payload = {'log_level': logger.getEffectiveLevel(), 'get_metadata': True} - fn_invoke_client = self._init_functions_invk_client(self.invoke_endpoint) + fn_invoke_client = self._init_functions_invk_client(invoke_endpoint) response = fn_invoke_client.invoke_function( - function_id=self.invoke_function_id, + function_id=function_id, invoke_function_body=json.dumps(payload, default=str), fn_invoke_type='sync' ) diff --git a/lithops/serverless/backends/singularity/singularity.py b/lithops/serverless/backends/singularity/singularity.py index 5a5408caf..773f3c488 100644 --- a/lithops/serverless/backends/singularity/singularity.py +++ b/lithops/serverless/backends/singularity/singularity.py @@ -216,6 +216,7 @@ def _generate_runtime_meta(self, singularity_image_name): logger.debug("Waiting for runtime metadata") + runtime_meta = None for i in range(0, 300): try: data_key = '/'.join([JOBS_PREFIX, runtime_name + '.meta']) diff --git a/lithops/standalone/backends/azure_vms/azure_vms.py b/lithops/standalone/backends/azure_vms/azure_vms.py index 4a7e01815..2b5feacaa 100644 --- a/lithops/standalone/backends/azure_vms/azure_vms.py +++ b/lithops/standalone/backends/azure_vms/azure_vms.py @@ -842,7 +842,8 @@ def __init__(self, name, config, compute_client=None, public=False): self.instance_data = None self.private_ip = None self.public_ip = '0.0.0.0' - self.home_dir = '/home/ubuntu' + ssh_user = self.config['ssh_username'] + self.home_dir = '/root' if ssh_user == 'root' else f'/home/{ssh_user}' self.ssh_credentials = { 'username': self.config['ssh_username'], @@ -1233,16 +1234,21 @@ def _delete_instance(self): def _stop_instance(self): """ - Stops the VM instance + Stops the VM instance. Azure keeps billing a VM that is only powered + off, so it is deallocated """ logger.debug(f"Stopping VM instance {self.name}") try: - self.compute_client.virtual_machines.begin_power_off( + self.compute_client.virtual_machines.begin_deallocate( self.config['resource_group'], self.name ) - except Exception: + except Exception as e: + # Inside the VM there may be no credentials allowed to deallocate it if os.path.isfile(SA_CONFIG_FILE): + logger.debug(f'Could not deallocate {self}, powering it off: {e}') os.system("shutdown -h now") + else: + logger.warning(f'Could not deallocate {self}: {e}') def stop(self): """ diff --git a/lithops/standalone/backends/azure_vms/config.py b/lithops/standalone/backends/azure_vms/config.py index c912664a2..12da90079 100644 --- a/lithops/standalone/backends/azure_vms/config.py +++ b/lithops/standalone/backends/azure_vms/config.py @@ -25,7 +25,7 @@ 'ssh_username': 'ubuntu', 'ssh_password': str(uuid.uuid4()), 'request_spot_instances': False, - 'delete_on_dismantle': True, + 'delete_on_dismantle': False, 'max_workers': 100, 'worker_processes': 'AUTO' } diff --git a/lithops/standalone/backends/vm/vm.py b/lithops/standalone/backends/vm/vm.py index 1ddb2e9ad..321e5d322 100644 --- a/lithops/standalone/backends/vm/vm.py +++ b/lithops/standalone/backends/vm/vm.py @@ -20,7 +20,7 @@ from lithops.standalone.utils import StandaloneMode from lithops.version import __version__ -from lithops.constants import COMPUTE_CLI_MSG +from lithops.constants import COMPUTE_CLI_MSG, CACHE_DIR from lithops.util.ssh_client import SSHClient, ssh_boot_status_message from lithops.standalone import LithopsValidationError @@ -37,6 +37,7 @@ def __init__(self, vm_config, mode): self.name = 'vm' self.config = vm_config self.mode = mode + self.cache_dir = os.path.join(CACHE_DIR, self.name) self.master = None logger.info(COMPUTE_CLI_MSG.format('Virtual Machine')) @@ -66,12 +67,26 @@ def list_images(self, **kwargs): def clean(self, **kwargs): pass - def clear(self, **kwargs): + def clear(self, job_keys=None): pass def dismantle(self, **kwargs): pass + def get_instance(self, name, **kwargs): + """ + Returns a VM class instance, which is how the master and its budget + keeper get hold of the VM they run on + """ + instance = VMInstance(self.config) + instance.name = name + + for key in kwargs: + if hasattr(instance, key) and kwargs[key] is not None: + setattr(instance, key, kwargs[key]) + + return instance + def get_worker_instance_type(self): return None @@ -85,9 +100,23 @@ class VMInstance: def __init__(self, config): self.config = config + self.name = 'lithops-consume' + self.instance_id = '0af1' + # Stored in redis by the master, which cannot store None + self.instance_type = 'VM' + # Lithops never created this machine, so it never deletes it + self.delete_on_dismantle = False self.public_ip = self.private_ip = self.config['ip_address'] self.ssh_client = None + ssh_user = self.config['ssh_username'] + self.home_dir = '/root' if ssh_user == 'root' else f'/home/{ssh_user}' + self.ssh_credentials = { + 'username': ssh_user, + 'password': self.config.get('ssh_password', None), + 'key_filename': self.config.get('ssh_key_filename', '~/.ssh/id_rsa') + } + logger.debug(f'{self} created') def __str__(self): @@ -98,12 +127,7 @@ def get_ssh_client(self): Creates an ssh client against the VM only if the Instance is the master """ if self.public_ip and not self.ssh_client: - ssh_credentials = { - 'username': self.config.get('ssh_user', 'root'), - 'password': self.config.get('ssh_password', None), - 'key_filename': self.config.get('ssh_key_filename', '~/.ssh/id_rsa') - } - self.ssh_client = SSHClient(self.public_ip, ssh_credentials) + self.ssh_client = SSHClient(self.public_ip, self.ssh_credentials) return self.ssh_client def del_ssh_client(self): @@ -151,13 +175,13 @@ def get_instance_data(self): """ Returns the instance information """ - return {'id': '0af1', 'public_ip': self.public_ip} + return {'id': self.instance_id, 'public_ip': self.public_ip} def get_instance_id(self): """ Requests the Instance ID """ - return "0af1" + return self.instance_id def get_public_ip(self): """ diff --git a/lithops/standalone/master.py b/lithops/standalone/master.py index 47262839f..eb3842056 100644 --- a/lithops/standalone/master.py +++ b/lithops/standalone/master.py @@ -335,12 +335,24 @@ def _worker_setup_script(standalone_handler, vm_data): return script +def _stop_failed_worker(worker): + """ + Stops, or deletes, a worker that could not be set up: nothing was + installed on it, not even the budget keeper that would stop it + """ + try: + logger.debug(f'Stopping {worker}, as it could not be set up') + worker.stop() + except Exception as e: + logger.error(f'Could not stop {worker}: {e}') + + def setup_worker_create_reuse(standalone_handler, worker_info, work_queue_name): """ Installs Lithops on a worker VM, recreating the instance when it does not come up or does not have what the runtime needs. The installation itself is left running in the background, and the worker reports back when its - service comes up + service comes up. A worker that cannot be set up is stopped """ worker = standalone_handler.backend.get_instance(**worker_info, public=False) @@ -349,6 +361,15 @@ def setup_worker_create_reuse(standalone_handler, worker_info, work_queue_name): save_worker(worker, standalone_handler.config) + try: + _install_worker(standalone_handler, worker, work_queue_name) + except Exception: + _stop_failed_worker(worker) + raise + + +def _install_worker(standalone_handler, worker, work_queue_name): + """Gets a registered worker VM up, validated and installing Lithops""" max_retries = worker.config.get( 'worker_create_retries', MAX_INSTANCE_CREATE_RETRIES ) diff --git a/lithops/standalone/standalone.py b/lithops/standalone/standalone.py index 7831c39bb..e57f44fac 100644 --- a/lithops/standalone/standalone.py +++ b/lithops/standalone/standalone.py @@ -114,22 +114,33 @@ def _make_request(self, method: str, endpoint: str, data=None): while the client has to go through the SSH connection it already has """ if self.is_lithops_worker: - return self._request_from_worker(method, endpoint, data) - return self._request_via_ssh(method, endpoint, data) + resp = self._request_from_worker(method, endpoint, data) + else: + resp = self._request_via_ssh(method, endpoint, data) + if isinstance(resp, dict) and 'error' in resp: + raise ValueError( + f'The master service could not handle {endpoint}: ' + f'{resp["error"]}' + ) + return resp def _request_from_worker(self, method: str, endpoint: str, data=None): """Calls the master service over HTTP, from inside the network""" url = self._master_url(endpoint) if method == 'GET': - resp = requests.get(url, timeout=1) - return resp.json() - if method == 'POST': + resp = requests.get(url, data=json.dumps(data), timeout=1) + elif method == 'POST': resp = requests.post(url, data=json.dumps(data)) + else: + raise ValueError(f'Unsupported HTTP method: {method}') + if not resp.content: resp.raise_for_status() - if not resp.content: - return None + return None + try: return resp.json() - raise ValueError(f'Unsupported HTTP method: {method}') + except ValueError: + resp.raise_for_status() + raise def _request_via_ssh(self, method: str, endpoint: str, data=None): """ @@ -364,6 +375,24 @@ def _acquire_workers(self, job_payload, required_workers: int): return new_workers, total_workers + def _stop_workers(self, workers): + """ + Stops the given workers, best effort, as a worker the master never + set up has no budget keeper to stop it. The ones a dismantle already + took out of the backend are not stopped again + """ + workers = [w for w in workers if w in self.backend.workers] + if not workers: + return + with cf.ThreadPoolExecutor(min(len(workers), 48)) as ex: + futures = {ex.submit(worker.stop): worker for worker in workers} + for future, worker in futures.items(): + try: + future.result() + except Exception as e: + logger.error(f'Could not stop {worker}: {e}') + self.backend.workers.remove(worker) + def _ensure_master_ready(self) -> None: """Sets the master service up unless it is already answering""" logger.debug(f"Checking if {self.backend.master} is ready") @@ -405,7 +434,12 @@ def invoke(self, job_payload): f'{total_calls} activations in {total_workers} workers' ) - self._ensure_master_ready() + try: + self._ensure_master_ready() + except Exception: + if self.exec_mode != StandaloneMode.CONSUME: + self._stop_workers(new_workers) + raise # The key never leaves the client: the master has its own to reach the # workers it creates @@ -467,16 +501,24 @@ def clean(self, **kwargs): def clear(self, job_keys=None, exception=None): """ - Stops the jobs this handler invoked. Workers meant to be reused stay - up, as the next job is going to run on them + Stops the given jobs this handler invoked, or all of them. Workers + meant to be reused stay up, as the next job is going to run on them, + and so do the workers of create mode while a job is left running, + since the backend can only stop all of them at once """ + if job_keys is None: + jobs_to_stop = list(self.jobs) + else: + jobs_to_stop = [job_key for job_key in self.jobs if job_key in job_keys] + try: - self._make_request('POST', 'job/stop', self.jobs) + self._make_request('POST', 'job/stop', jobs_to_stop) logger.debug('Jobs stopped on the master') except Exception as e: logger.debug(f'Could not stop the jobs on the master: {e}') + self.jobs = [job_key for job_key in self.jobs if job_key not in jobs_to_stop] - if self.exec_mode != StandaloneMode.REUSE: + if self.exec_mode != StandaloneMode.REUSE and not self.jobs: self.backend.clear(job_keys) def list_jobs(self): diff --git a/lithops/tests/test_aws_backends.py b/lithops/tests/test_aws_backends.py new file mode 100644 index 000000000..169c1b60a --- /dev/null +++ b/lithops/tests/test_aws_backends.py @@ -0,0 +1,522 @@ +# +# Unit tests for the AWS Lambda and AWS Batch backend internals. No AWS account +# required: the boto3 clients are botocore stubs or mocks. +# + +import re +from unittest.mock import MagicMock + +import pytest + +boto3 = pytest.importorskip('boto3') +botocore = pytest.importorskip('botocore') + +from botocore.credentials import Credentials # noqa: E402 +from botocore.exceptions import ClientError, ConnectTimeoutError, EndpointConnectionError, ReadTimeoutError # noqa: E402 +from botocore.stub import ANY, Stubber # noqa: E402 + +from lithops.serverless.backends.aws_batch import config as batch_config # noqa: E402 +from lithops.serverless.backends.aws_batch.aws_batch import AWSBatchBackend # noqa: E402 +from lithops.serverless.backends.aws_lambda import aws_lambda as aws_lambda_module # noqa: E402 +from lithops.serverless.backends.aws_lambda import config as lambda_config # noqa: E402 +from lithops.serverless.backends.aws_lambda.aws_lambda import AWSLambdaBackend # noqa: E402 +from lithops.version import __version__ # noqa: E402 + +REGION = 'us-east-1' +USER_KEY = 'abcd' +CURRENT_PACKAGE = f'lithops_v{__version__.replace(".", "")}_{USER_KEY}' +OK = {'ResponseMetadata': {'HTTPStatusCode': 200}} + + +def _stubbed_client(service): + client = boto3.client( + service, region_name=REGION, aws_access_key_id='AKIDEXAMPLE', aws_secret_access_key='secret' + ) + return client, Stubber(client) + + +# --------------------------------------------------------------------------- +# AWS Batch +# --------------------------------------------------------------------------- + +def _batch_load_config(**overrides): + cfg = {'aws_batch': {'execution_role': 'arn:role', 'security_groups': ['sg'], 'region': REGION, **overrides}} + batch_config.load_config(cfg) + return cfg['aws_batch'] + + +def _make_batch_backend(namespace=None, **overrides): + """Build an AWSBatchBackend without running __init__ (which calls STS).""" + backend = AWSBatchBackend.__new__(AWSBatchBackend) + backend.name = 'aws_batch' + backend.region = REGION + backend.aws_batch_config = { + 'runtime_cpu': 0.5, 'runtime_memory': 1024, 'execution_role': 'arn:exec', 'job_role': 'arn:job', + 'assign_public_ip': True, 'env_type': 'FARGATE_SPOT', **overrides + } + backend.env_type = backend.aws_batch_config['env_type'] + backend.namespace = namespace + backend.account_id = '123456789012' + backend.package = f'{CURRENT_PACKAGE}_{namespace}' if namespace else CURRENT_PACKAGE + backend._queue_name = f'{backend.package}_queue' + backend.batch_client = MagicMock() + return backend + + +class TestBatchMaxWorkers: + + def test_default_max_workers_is_int(self): + cfg = _batch_load_config() + assert cfg['max_workers'] == 20 + assert isinstance(cfg['max_workers'], int) + + def test_array_job_size_is_int(self): + backend = _make_batch_backend() + backend.batch_client, stubber = _stubbed_client('batch') + stubber.add_response( + 'submit_job', {'jobName': 'job', 'jobId': 'id'}, + {'jobName': ANY, 'jobQueue': ANY, 'jobDefinition': ANY, + 'arrayProperties': {'size': 20}, 'containerOverrides': ANY} + ) + payload = {'executor_id': 'ex', 'job_id': 'A000', 'job_key': 'ex-A000', + 'total_calls': 100, 'max_workers': 20.0, 'chunksize': 1} + with stubber: + backend.invoke('rt:latest', 1024, payload) + stubber.assert_no_pending_responses() + assert payload['chunksize'] == 5 and isinstance(payload['chunksize'], int) + + +class TestBatchJobDefinition: + + def _register_args(self, backend, runtime_memory): + backend.batch_client.describe_job_definitions.return_value = {**OK, 'jobDefinitions': []} + backend.batch_client.register_job_definition.return_value = OK + backend._create_job_def('rt:latest', runtime_memory) + return backend.batch_client.register_job_definition.call_args.kwargs + + def test_uses_requested_memory(self): + backend = _make_batch_backend(runtime_cpu=1, runtime_memory=2048) + kwargs = self._register_args(backend, 4096) + resources = {r['type']: r['value'] for r in kwargs['containerProperties']['resourceRequirements']} + assert resources == {'VCPU': '1', 'MEMORY': '4096'} + assert kwargs['jobDefinitionName'].endswith('--4096mb') + + def test_fargate_rejects_invalid_requested_memory(self): + backend = _make_batch_backend(runtime_cpu=0.5, runtime_memory=1024) + with pytest.raises(Exception, match='runtime memory is not valid'): + self._register_args(backend, 1536) + backend.batch_client.register_job_definition.assert_not_called() + + def test_ec2_does_not_check_fargate_memory(self): + backend = _make_batch_backend(env_type='EC2', runtime_cpu=1, runtime_memory=1024) + kwargs = self._register_args(backend, 1536) + assert kwargs['platformCapabilities'] == ['EC2'] + + +class TestBatchJobDefinitionName: + + @pytest.mark.parametrize('namespace', [None, 'dev']) + def test_current_version_name_is_unchanged(self, namespace): + backend = _make_batch_backend(namespace=namespace) + package = f'{CURRENT_PACKAGE}_{namespace}' if namespace else CURRENT_PACKAGE + name = backend._format_jobdef_name('batch-default-runtime-v310:latest', 1024) + assert name == f'{package}--FARGATE_SPOT--batch-default-runtime-v310--latest--1024mb' + + @pytest.mark.parametrize('namespace', [None, 'dev']) + def test_requested_version_is_used(self, namespace): + backend = _make_batch_backend(namespace=namespace) + name = backend._format_jobdef_name('rt:latest', 1024, '3.7.0') + suffix = '_dev' if namespace else '' + assert name == f'lithops_v370_{USER_KEY}{suffix}--FARGATE_SPOT--rt--latest--1024mb' + + @pytest.mark.parametrize('runtime_name', ['rt:latest', 'my-rt:v1_0', 'batch-default-runtime-v310:latest']) + def test_round_trip(self, runtime_name): + backend = _make_batch_backend(namespace='dev') + name = backend._format_jobdef_name(runtime_name, 2048) + assert re.fullmatch(r'[A-Za-z0-9_-]{1,128}', name) + assert backend._unformat_jobdef_name(name) == (runtime_name, 2048, __version__) + + def test_list_runtimes_rows_match_deployed_keys(self): + backend = _make_batch_backend() + name = backend._format_jobdef_name('rt:latest', 1024) + backend.batch_client.describe_job_definitions.return_value = { + **OK, 'jobDefinitions': [{'jobDefinitionName': name, 'jobDefinitionArn': 'arn:jd', 'status': 'ACTIVE'}] + } + rows = backend.list_runtimes('rt:latest') + assert rows == [('rt:latest', 1024, __version__, name)] + rt_name, rt_memory, version, _ = rows[0] + assert backend.get_runtime_key(rt_name, rt_memory, version) == backend.get_runtime_key('rt:latest', 1024) + + def test_delete_runtime_finds_listed_job_definition(self): + backend = _make_batch_backend() + name = backend._format_jobdef_name('rt:latest', 1024) + backend.batch_client.describe_job_definitions.return_value = { + **OK, 'jobDefinitions': [{'jobDefinitionName': name, 'jobDefinitionArn': 'arn:jd', 'status': 'ACTIVE'}] + } + backend.batch_client.deregister_job_definition.return_value = OK + rt_name, rt_memory, version, _ = backend.list_runtimes()[0] + backend.delete_runtime(rt_name, rt_memory, version) + backend.batch_client.deregister_job_definition.assert_called_once_with(jobDefinition='arn:jd') + + def test_delete_runtime_of_missing_job_definition(self): + backend = _make_batch_backend() + backend.batch_client.describe_job_definitions.return_value = {**OK, 'jobDefinitions': []} + backend.delete_runtime('rt:latest', 1024) + backend.batch_client.deregister_job_definition.assert_not_called() + + def test_job_definitions_past_the_first_page(self): + backend = _make_batch_backend() + backend.batch_client, stubber = _stubbed_client('batch') + names = [backend._format_jobdef_name(f'rt{i}:latest', 1024) for i in range(2)] + pages = [ + {**OK, 'jobDefinitions': [{'jobDefinitionName': names[0], 'jobDefinitionArn': 'arn:jd0', + 'revision': 1, 'type': 'container'}], 'nextToken': 't1'}, + {**OK, 'jobDefinitions': [{'jobDefinitionName': names[1], 'jobDefinitionArn': 'arn:jd1', + 'revision': 1, 'type': 'container'}]}, + ] + stubber.add_response('describe_job_definitions', pages[0], {'status': 'ACTIVE'}) + stubber.add_response('describe_job_definitions', pages[1], {'status': 'ACTIVE', 'nextToken': 't1'}) + with stubber: + assert [row[0] for row in backend.list_runtimes()] == ['rt0:latest', 'rt1:latest'] + stubber.assert_no_pending_responses() + + +# --------------------------------------------------------------------------- +# AWS Lambda +# --------------------------------------------------------------------------- + +def _lambda_load_config(**overrides): + cfg = {'aws_lambda': {'execution_role': 'arn:role', 'region': REGION, **overrides}} + lambda_config.load_config(cfg) + return cfg['aws_lambda'] + + +def _make_lambda_backend(**overrides): + """Build an AWSLambdaBackend without running __init__ (which calls STS).""" + backend = AWSLambdaBackend.__new__(AWSLambdaBackend) + backend.name = 'aws_lambda' + backend.region = REGION + backend.lambda_config = _lambda_load_config(**overrides) + backend.role_arn = 'arn:role' + backend.namespace = None + backend.user_key = USER_KEY + backend.account_id = '123456789012' + backend.package = CURRENT_PACKAGE + backend.user_agent = 'lithops-test' + backend.host = f'lambda.{REGION}.amazonaws.com' + backend.credentials = Credentials('AKIDEXAMPLE', 'secret') + backend.session = MagicMock() + backend.lambda_client = MagicMock() + backend.ecr_client = MagicMock() + backend.lambda_client.create_function.return_value = {'ResponseMetadata': {'HTTPStatusCode': 201}} + backend.lambda_client.get_function.return_value = { + 'Configuration': {'State': 'Active', 'LastUpdateStatus': 'Successful'} + } + return backend + + +def _conflict(): + return ClientError( + {'Error': {'Code': 'ResourceConflictException', 'Message': 'Function already exist'}}, 'CreateFunction' + ) + + +class _TooManyAttempts(BaseException): + """Stops a retry loop that would otherwise never end.""" + + +class TestLambdaInvokeRetries: + + @pytest.fixture(autouse=True) + def no_sleep(self, monkeypatch): + monkeypatch.setattr(aws_lambda_module.time, 'sleep', lambda s: None) + + def _send(self, backend, outcomes): + calls = [] + + def send(prepared): + calls.append(prepared.headers['Authorization']) + if len(calls) > 100: + raise _TooManyAttempts() + outcome = outcomes[min(len(calls), len(outcomes)) - 1] + if isinstance(outcome, Exception): + raise outcome + return outcome + + backend.session.send.side_effect = send + return calls + + @staticmethod + def _accepted(): + return MagicMock(status_code=202, headers={'x-amzn-RequestId': 'req-1'}) + + @pytest.mark.parametrize('error', [ + EndpointConnectionError(endpoint_url='u'), + ConnectTimeoutError(endpoint_url='u'), + ]) + def test_gives_up_after_last_attempt(self, error): + backend = _make_lambda_backend() + calls = self._send(backend, [error]) + with pytest.raises(type(error)): + backend.invoke('rt', 256, {'a': 1}) + assert len(calls) == aws_lambda_module.INVOKE_RETRIES + + def test_resigns_each_attempt(self, monkeypatch): + backend = _make_lambda_backend() + add_auth = MagicMock(side_effect=lambda request: request.headers.__setitem__('Authorization', 'sig')) + monkeypatch.setattr(aws_lambda_module, 'SigV4Auth', MagicMock(return_value=MagicMock(add_auth=add_auth))) + error = EndpointConnectionError(endpoint_url='u') + calls = self._send(backend, [error, error, self._accepted()]) + assert backend.invoke('rt', 256, {'a': 1}) == 'req-1' + assert len(calls) == 3 + assert add_auth.call_count == 3 + + def test_read_timeout_is_not_resent(self): + backend = _make_lambda_backend() + calls = self._send(backend, [ReadTimeoutError(endpoint_url='u'), self._accepted()]) + with pytest.raises(ReadTimeoutError): + backend.invoke('rt', 256, {'a': 1}) + assert len(calls) == 1 + + +class TestLambdaRuntimeUpdate: + + def test_default_runtime_updates_existing_function(self, monkeypatch): + backend = _make_lambda_backend(env_vars={'A': 'b'}) + monkeypatch.setattr(backend, '_get_layer', lambda runtime_name: 'arn:layer') + monkeypatch.setattr(backend, '_create_handler_bin', lambda: b'zip') + backend.lambda_client.create_function.side_effect = _conflict() + + backend._deploy_default_runtime('default-runtime-v311', 256, 600) + + function_name = backend._format_function_name('default-runtime-v311', 256) + backend.lambda_client.update_function_code.assert_called_once_with(FunctionName=function_name, ZipFile=b'zip') + conf = backend.lambda_client.update_function_configuration.call_args.kwargs + assert conf['FunctionName'] == function_name + assert conf['Timeout'] == 600 + assert conf['MemorySize'] == 256 + assert conf['Layers'] == ['arn:layer'] + assert conf['Environment'] == {'Variables': {'A': 'b'}} + + def test_container_runtime_updates_existing_function(self): + backend = _make_lambda_backend() + backend.ecr_client.describe_images.return_value = { + 'imageDetails': [{'imageTags': ['v2'], 'imageDigest': 'sha256:new'}] + } + backend.lambda_client.create_function.side_effect = _conflict() + + backend._deploy_container_runtime('myrt:v2', 1024, 300) + + code = backend.lambda_client.update_function_code.call_args.kwargs + assert code['ImageUri'].endswith('@sha256:new') + assert code['Architectures'] == ['x86_64'] + conf = backend.lambda_client.update_function_configuration.call_args.kwargs + assert conf['Timeout'] == 300 + assert 'Architectures' not in conf and 'PackageType' not in conf + + def test_waits_for_update_before_configuring(self, monkeypatch): + monkeypatch.setattr(aws_lambda_module.time, 'sleep', lambda s: None) + backend = _make_lambda_backend() + backend.ecr_client.describe_images.return_value = { + 'imageDetails': [{'imageTags': ['latest'], 'imageDigest': 'sha256:new'}] + } + backend.lambda_client.create_function.side_effect = _conflict() + events = [] + statuses = iter(['Successful', 'InProgress', 'Successful', 'Successful']) + + def get_function(FunctionName): + status = next(statuses) + events.append(f'get:{status}') + return {'Configuration': {'State': 'Active', 'LastUpdateStatus': status}} + + backend.lambda_client.get_function.side_effect = get_function + backend.lambda_client.update_function_code.side_effect = lambda **kw: events.append('code') + def update_configuration(**kw): + events.append('config') + return {'FunctionArn': 'arn:fn'} + + backend.lambda_client.update_function_configuration.side_effect = update_configuration + backend._deploy_container_runtime('myrt', 1024, 300) + assert events == ['get:Successful', 'code', 'get:InProgress', 'get:Successful', 'config', 'get:Successful'] + + def test_failed_update_raises(self): + backend = _make_lambda_backend() + backend.lambda_client.get_function.return_value = { + 'Configuration': {'State': 'Active', 'LastUpdateStatus': 'Failed'} + } + with pytest.raises(Exception, match='not updated'): + backend._wait_for_function_deployed('fn') + + +class TestLambdaEnvVarsAndUserTags: + + @pytest.mark.parametrize('key', ['env_vars', 'user_tags']) + @pytest.mark.parametrize('value', [ + {'A': 'b', 'N': 1}, + [{'name': 'A', 'value': 'b'}, {'name': 'N', 'value': 1}], + ]) + def test_dict_and_list_forms_are_accepted(self, key, value): + assert _lambda_load_config(**{key: value})[key] == {'A': 'b', 'N': '1'} + + @pytest.mark.parametrize('key', ['env_vars', 'user_tags']) + def test_empty_values(self, key): + assert _lambda_load_config()[key] == {} + assert _lambda_load_config(**{key: None})[key] == {} + + @pytest.mark.parametrize('key', ['env_vars', 'user_tags']) + @pytest.mark.parametrize('value', ['A=b', [{'name': 'A'}], ['A']]) + def test_invalid_forms_are_rejected(self, key, value): + with pytest.raises(Exception, match=key): + _lambda_load_config(**{key: value}) + + @pytest.mark.parametrize('env_vars', [{'A': 'b'}, [{'name': 'A', 'value': 'b'}]]) + @pytest.mark.parametrize('user_tags', [{'team': 'x'}, [{'name': 'team', 'value': 'x'}]]) + def test_container_runtime_receives_both(self, env_vars, user_tags): + backend = _make_lambda_backend(env_vars=env_vars, user_tags=user_tags) + backend.ecr_client.describe_images.return_value = { + 'imageDetails': [{'imageTags': ['latest'], 'imageDigest': 'sha256:abc'}] + } + backend._deploy_container_runtime('myrt', 1024, 300) + kwargs = backend.lambda_client.create_function.call_args.kwargs + assert kwargs['Environment'] == {'Variables': {'A': 'b'}} + assert kwargs['Tags'] == {'runtime_name': 'myrt', 'lithops_version': __version__, 'team': 'x'} + + @pytest.mark.parametrize('env_vars', [{'A': 'b'}, [{'name': 'A', 'value': 'b'}]]) + def test_default_runtime_receives_env_vars(self, env_vars, monkeypatch): + backend = _make_lambda_backend(env_vars=env_vars) + monkeypatch.setattr(backend, '_get_layer', lambda runtime_name: 'arn:layer') + monkeypatch.setattr(backend, '_create_handler_bin', lambda: b'zip') + backend._deploy_default_runtime('default-runtime-v311', 256, 600) + kwargs = backend.lambda_client.create_function.call_args.kwargs + assert kwargs['Environment'] == {'Variables': {'A': 'b'}} + + +class TestLambdaLayerDeletion: + + def test_paginated_layer_versions_are_all_deleted(self): + backend = _make_lambda_backend() + backend.lambda_client, stubber = _stubbed_client('lambda') + stubber.add_response( + 'list_layer_versions', {'LayerVersions': [{'Version': 51}], 'NextMarker': 'm1'}, {'LayerName': 'ly'} + ) + stubber.add_response( + 'list_layer_versions', {'LayerVersions': [{'Version': 1}]}, {'LayerName': 'ly', 'Marker': 'm1'} + ) + deleted = {'ResponseMetadata': {'HTTPStatusCode': 204}} + stubber.add_response('delete_layer_version', deleted, {'LayerName': 'ly', 'VersionNumber': 51}) + stubber.add_response('delete_layer_version', deleted, {'LayerName': 'ly', 'VersionNumber': 1}) + with stubber: + backend._delete_layer('ly') + stubber.assert_no_pending_responses() + + def test_layers_past_the_first_page_are_listed(self): + backend = _make_lambda_backend() + backend.lambda_client, stubber = _stubbed_client('lambda') + + def layer(name): + arn = f'arn:aws:lambda:{REGION}:123456789012:layer:{name}:1' + return {'LayerName': name, 'LatestMatchingVersion': {'LayerVersionArn': arn}} + + names = [f'lithops_v3_{USER_KEY}_{i}' for i in range(2)] + stubber.add_response('list_layers', {'Layers': [layer(names[0]), layer('other')], 'NextMarker': 'm1'}, {}) + stubber.add_response('list_layers', {'Layers': [layer(names[1])]}, {'Marker': 'm1'}) + with stubber: + assert [name for name, _ in backend._list_layers()] == names + stubber.assert_no_pending_responses() + + +class TestLambdaRuntimeListing: + + @staticmethod + def _function(name): + arn = f'arn:aws:lambda:{REGION}:123456789012:function:{name}' + return {'FunctionName': name, 'MemorySize': 256, 'FunctionArn': arn} + + def test_functions_past_the_first_page_are_listed(self): + backend = _make_lambda_backend() + backend.lambda_client, stubber = _stubbed_client('lambda') + names = [f'lithops-worker-{USER_KEY}-{i}' for i in range(2)] + pages = [ + ({'Functions': [self._function(names[0]), self._function('other')], 'NextMarker': 'm1'}, {}), + ({'Functions': [self._function(names[1])]}, {'Marker': 'm1'}), + ] + for i, (page, params) in enumerate(pages): + stubber.add_response('list_functions', page, params) + stubber.add_response( + 'list_tags', {'Tags': {'runtime_name': f'rt{i}', 'lithops_version': __version__}}, + {'Resource': self._function(names[i])['FunctionArn']} + ) + with stubber: + rows = backend.list_runtimes() + stubber.assert_no_pending_responses() + assert rows == [(f'rt{i}', 256, __version__, name) for i, name in enumerate(names)] + + def test_clean_deletes_functions_past_the_first_page(self, monkeypatch): + backend = _make_lambda_backend() + backend.lambda_client, stubber = _stubbed_client('lambda') + monkeypatch.setattr(backend, '_list_layers', lambda: []) + names = [f'lithops-worker-{USER_KEY}-{i}' for i in range(2)] + deleted = {'ResponseMetadata': {'HTTPStatusCode': 204}} + stubber.add_response('list_functions', {'Functions': [self._function(names[0])], 'NextMarker': 'm1'}, {}) + stubber.add_response('delete_function', deleted, {'FunctionName': names[0]}) + stubber.add_response('list_functions', {'Functions': [self._function(names[1])]}, {'Marker': 'm1'}) + stubber.add_response('delete_function', deleted, {'FunctionName': names[1]}) + with stubber: + backend.clean() + stubber.assert_no_pending_responses() + + def test_delete_runtime_of_missing_function(self, monkeypatch): + backend = _make_lambda_backend() + backend.lambda_client.delete_function.side_effect = ClientError( + {'Error': {'Code': 'ResourceNotFoundException', 'Message': 'Function not found'}}, 'DeleteFunction' + ) + delete_layer = MagicMock() + monkeypatch.setattr(backend, '_delete_layer', delete_layer) + backend.delete_runtime(backend._get_default_runtime_name(), 256) + delete_layer.assert_called_once() + + def test_delete_function_raises_other_errors(self): + backend = _make_lambda_backend() + backend.lambda_client.delete_function.side_effect = ClientError( + {'Error': {'Code': 'AccessDeniedException', 'Message': 'denied'}}, 'DeleteFunction' + ) + with pytest.raises(ClientError): + backend._delete_function('fn') + + +class TestLambdaTags: + + USER_TAGS = {'team': 'x'} + + def _expected(self, runtime_name): + return {'runtime_name': runtime_name, 'lithops_version': __version__, **self.USER_TAGS} + + @pytest.fixture + def default_backend(self, monkeypatch): + backend = _make_lambda_backend(user_tags=self.USER_TAGS) + monkeypatch.setattr(backend, '_get_layer', lambda runtime_name: 'arn:layer') + monkeypatch.setattr(backend, '_create_handler_bin', lambda: b'zip') + return backend + + def test_default_runtime_is_created_with_user_tags(self, default_backend): + default_backend._deploy_default_runtime('default-runtime-v311', 256, 600) + tags = default_backend.lambda_client.create_function.call_args.kwargs['Tags'] + assert tags == self._expected('default-runtime-v311') + + def test_default_runtime_update_sets_tags(self, default_backend): + client = default_backend.lambda_client + client.create_function.side_effect = _conflict() + client.update_function_configuration.return_value = {'FunctionArn': 'arn:fn'} + default_backend._deploy_default_runtime('default-runtime-v311', 256, 600) + client.tag_resource.assert_called_once_with(Resource='arn:fn', Tags=self._expected('default-runtime-v311')) + client.untag_resource.assert_not_called() + + def test_container_runtime_update_sets_tags(self): + backend = _make_lambda_backend(user_tags=self.USER_TAGS) + backend.ecr_client.describe_images.return_value = { + 'imageDetails': [{'imageTags': ['latest'], 'imageDigest': 'sha256:abc'}] + } + backend.lambda_client.create_function.side_effect = _conflict() + backend.lambda_client.update_function_configuration.return_value = {'FunctionArn': 'arn:fn'} + backend._deploy_container_runtime('myrt', 1024, 300) + backend.lambda_client.tag_resource.assert_called_once_with(Resource='arn:fn', Tags=self._expected('myrt')) diff --git a/lithops/tests/test_azure_aliyun_oracle_backends.py b/lithops/tests/test_azure_aliyun_oracle_backends.py new file mode 100644 index 000000000..f5802565d --- /dev/null +++ b/lithops/tests/test_azure_aliyun_oracle_backends.py @@ -0,0 +1,310 @@ +# +# Unit tests for the Azure Functions, Azure Container Apps, Aliyun FC and Oracle Functions +# backend internals. No cloud account required; the SDKs and the az CLI are replaced by fakes. +# + +import json +import os +import types + +import pytest + +from lithops.version import __version__ + + +class FakeClock: + """Replaces the time module of a backend; fails instead of hanging in an endless loop.""" + + def __init__(self, max_sleeps=10000): + self.now = 0.0 + self.sleeps = 0 + self.max_sleeps = max_sleeps + + def time(self): + return self.now + + def sleep(self, seconds): + self.sleeps += 1 + if self.sleeps > self.max_sleeps: + raise AssertionError('Loop without deadline') + self.now += seconds + + +# /---------------------------------------------------------------------------/ +# Azure Functions +# /---------------------------------------------------------------------------/ + +def _azure_functions_module(): + pytest.importorskip('azure.core') + pytest.importorskip('azure.storage.queue') + from lithops.serverless.backends.azure_functions import azure_functions + return azure_functions + + +def _azure_functions_backend(trigger='pub/sub'): + af = _azure_functions_module() + backend = af.AzureFunctionAppBackend.__new__(af.AzureFunctionAppBackend) + backend.name = 'azure_functions' + backend.trigger = trigger + backend.resource_group = 'rg' + backend.storage_account_name = 'storageacc' + backend.location = 'westeurope' + backend.functions_version = 4 + backend.af_config = {'runtime_timeout': 300, 'runtime_memory': 2048, 'trigger': trigger} + return af, backend + + +class FakeHTTPSConnection: + status = 200 + reason = 'OK' + body = 'invocation-id' + + def __init__(self, netloc, context=None): + pass + + def request(self, method, route, body=None): + pass + + def getresponse(self): + return types.SimpleNamespace( + status=self.status, reason=self.reason, read=lambda: self.body.encode('utf-8') + ) + + def close(self): + pass + + +@pytest.mark.parametrize('status, reason', [(404, 'Not Found'), (500, 'Internal Server Error')]) +def test_azure_functions_https_invoke_error_raises(monkeypatch, status, reason): + af, backend = _azure_functions_backend(trigger='https') + conn = type('Conn', (FakeHTTPSConnection,), {'status': status, 'reason': reason, 'body': 'boom'}) + monkeypatch.setattr(af.http.client, 'HTTPSConnection', conn) + + with pytest.raises(Exception, match=f'{status}.*boom'): + backend.invoke('rt', 2048, {'call_ids': ['00000']}) + + +def test_azure_functions_https_invoke_success_and_throttle(monkeypatch): + af, backend = _azure_functions_backend(trigger='https') + monkeypatch.setattr(af, 'time', FakeClock()) + + monkeypatch.setattr(af.http.client, 'HTTPSConnection', FakeHTTPSConnection) + assert backend.invoke('rt', 2048, {}) == 'invocation-id' + + throttled = type('Conn', (FakeHTTPSConnection,), {'status': 429, 'reason': 'Too Many Requests'}) + monkeypatch.setattr(af.http.client, 'HTTPSConnection', throttled) + assert backend.invoke('rt', 2048, {}) is None + + +def test_azure_functions_pubsub_metadata_wait_has_deadline(monkeypatch): + af, backend = _azure_functions_backend(trigger='pub/sub') + backend.af_config['runtime_timeout'] = 30 + clock = FakeClock() + monkeypatch.setattr(af, 'time', clock) + + queue = types.SimpleNamespace( + send_message=lambda msg: types.SimpleNamespace(id='msg-id'), + receive_message=lambda: None, + clear_messages=lambda: None, + ) + backend.queue_service = types.SimpleNamespace(get_queue_client=lambda name: queue) + + with pytest.raises(Exception, match='No response from function .* after 30 seconds'): + backend.invoke('rt', 2048, {'get_metadata': True}, return_result=True) + assert clock.now <= 32 + + +def _az_list_response(*apps): + return [{'Name': name, 'Tags': tags} for name, tags in apps] + + +def test_azure_functions_list_runtimes_by_runtime_name(monkeypatch): + _, backend = _azure_functions_backend() + app_name = backend._format_function_name('my-runtime', 4096) + response = _az_list_response( + (app_name, {'type': 'lithops-runtime', 'lithops_version': __version__, + 'runtime_name': 'my-runtime', 'runtime_memory': '4096'}), + ('lithops-worker-other', {'type': 'lithops-runtime', 'lithops_version': __version__, + 'runtime_name': 'other', 'runtime_memory': '512'}), + ('not-lithops', {'type': 'web'}), + ) + monkeypatch.setattr(backend, '_run_az_command', lambda cmd, **kw: response) + + assert backend.list_runtimes('my-runtime') == [('my-runtime', 4096, __version__, app_name)] + assert len(backend.list_runtimes('all')) == 2 + assert backend.list_runtimes(app_name) == [] + + +def test_azure_functions_runtime_memory_selects_app(monkeypatch, tmp_path): + af, backend = _azure_functions_backend(trigger='pub/sub') + + assert backend.get_runtime_key('rt', 2048) != backend.get_runtime_key('rt', 4096) + # Requested sizes that map to the same Flex instance memory share the app + assert backend.get_runtime_key('rt', 1024) == backend.get_runtime_key('rt', 2048) + default_name = backend._format_function_name('default-runtime-v314', 4096, '3.6.5.dev0') + assert len(default_name) <= 60 + + monkeypatch.setattr(af.config, 'BUILD_DIR', str(tmp_path)) + os.makedirs(os.path.join(backend._format_build_dir('rt'), af.config.ACTION_DIR)) + + commands, published = [], [] + monkeypatch.setattr(backend, '_function_app_exists', lambda name: True) + monkeypatch.setattr(backend, '_run_az_command', lambda cmd, **kw: commands.append(cmd)) + monkeypatch.setattr(backend, '_ensure_queue', lambda name: None) + + def publish(function_name, build_dir): + fn_file = os.path.join(build_dir, af.config.ACTION_DIR, 'function.json') + with open(fn_file) as f: + bindings = json.load(f)['bindings'] + published.append((function_name, bindings[0]['queueName'])) + monkeypatch.setattr(backend, '_publish_function', publish) + + backend._create_function('rt', 2048, 300) + backend._create_function('rt', 4096, 300) + + (name_2048, queue_2048), (name_4096, queue_4096) = published + assert name_2048 == backend._format_function_name('rt', 2048) + assert name_4096 == backend._format_function_name('rt', 4096) + assert name_2048 != name_4096 + assert queue_2048 == backend._format_queue_name(name_2048, af.config.IN_QUEUE) + assert queue_4096 == backend._format_queue_name(name_4096, af.config.IN_QUEUE) + assert queue_2048 != queue_4096 + + +def test_azure_functions_create_tags_runtime_memory(monkeypatch, tmp_path): + af, backend = _azure_functions_backend(trigger='https') + monkeypatch.setattr(af.config, 'BUILD_DIR', str(tmp_path)) + os.makedirs(os.path.join(backend._format_build_dir('rt'), af.config.ACTION_DIR)) + + commands = [] + monkeypatch.setattr(backend, '_function_app_exists', lambda name: False) + monkeypatch.setattr(backend, '_run_az_command', lambda cmd, **kw: commands.append(cmd)) + monkeypatch.setattr(backend, '_publish_function', lambda name, build_dir: None) + + backend._create_function('rt', 3000, 300) + + create_cmd = next(cmd for cmd in commands if cmd.startswith('az functionapp create')) + assert '--instance-memory 4096' in create_cmd + assert 'runtime_memory=4096' in create_cmd + + +def test_azure_functions_delete_legacy_app_by_tags(monkeypatch): + _, backend = _azure_functions_backend() + legacy_name = 'lithops-worker-rt-365dev0-0123456789ab' + response = _az_list_response( + (legacy_name, {'type': 'lithops-runtime', 'lithops_version': __version__, 'runtime_name': 'rt'}), + ) + commands = [] + + def run_az(cmd, return_json=False, **kw): + if return_json: + return response + commands.append(cmd) + monkeypatch.setattr(backend, '_run_az_command', run_az) + backend.queue_service = types.SimpleNamespace(delete_queue=lambda name: None) + + assert backend.list_runtimes('rt') == [('rt', 2048, __version__, legacy_name)] + backend.delete_runtime('rt', 2048, __version__) + assert commands == [f'az functionapp delete --name {legacy_name} --resource-group rg'] + + +# /---------------------------------------------------------------------------/ +# Azure Container Apps +# /---------------------------------------------------------------------------/ + +def test_azure_containers_list_runtimes_by_runtime_name(monkeypatch): + pytest.importorskip('azure.storage.queue') + from lithops.serverless.backends.azure_containers import azure_containers as ac + + backend = ac.AzureContainerAppBackend.__new__(ac.AzureContainerAppBackend) + backend.storage_account_name = 'storageacc' + backend.trigger = 'pub/sub' + image = 'docker.io/user/lithops-azurecontainers-default:latest' + app_name = backend._format_containerapp_name(image, 1024) + response = [ + {'Name': app_name, 'Tags': {'type': 'lithops-runtime', 'lithops_version': __version__, + 'runtime_name': image, 'runtime_memory': '1024'}}, + {'Name': 'lithops-worker-other', 'Tags': {'type': 'lithops-runtime', 'lithops_version': __version__, + 'runtime_name': 'other:latest', 'runtime_memory': '512'}}, + ] + monkeypatch.setattr(backend, '_run_az_command', lambda cmd, **kw: response) + + runtimes = backend.list_runtimes(image) + assert runtimes == [(image, 1024, __version__, app_name)] + assert runtimes[0][1] in ac.config.ALLOWED_MEM + assert len(backend.list_runtimes('all')) == 2 + + +# /---------------------------------------------------------------------------/ +# Aliyun Function Compute +# /---------------------------------------------------------------------------/ + +def test_aliyun_fc_metadata_retry_resends_payload(monkeypatch): + pytest.importorskip('alibabacloud_fc20230330') + from lithops.serverless.backends.aliyun_fc import aliyun_fc + + backend = aliyun_fc.AliyunFunctionComputeBackend.__new__(aliyun_fc.AliyunFunctionComputeBackend) + backend.deploy_mode = 'zip' + monkeypatch.setattr(aliyun_fc, 'time', FakeClock()) + + bodies = [] + meta = {'preinstalls': [], 'python_version': '3.12'} + + def invoke_function_with_options(function_name, request, headers, runtime): + bodies.append(request.body.read()) + if len(bodies) == 1: + raise aliyun_fc.FCClientException(status_code=412, code='PreconditionFailed', message='pending') + return types.SimpleNamespace(body=json.dumps(meta).encode('utf-8'), headers={}) + backend.fc_client = types.SimpleNamespace(invoke_function_with_options=invoke_function_with_options) + + assert backend._generate_runtime_meta('lithops-worker-fn', max_attempts=3, retry_sleep=1) == meta + assert len(bodies) == 2 + assert bodies[0] == bodies[1] + assert json.loads(bodies[1])['get_metadata'] is True + + +# /---------------------------------------------------------------------------/ +# Oracle Functions +# /---------------------------------------------------------------------------/ + +def test_oracle_invoke_uses_function_of_its_runtime_memory(monkeypatch): + pytest.importorskip('oci') + from lithops.serverless.backends.oracle_f import oracle_f + + backend = oracle_f.OracleCloudFunctionsBackend.__new__(oracle_f.OracleCloudFunctionsBackend) + backend.config = {} + backend.user = 'ocid1.user.oc1..aaaa' + backend.region = 'eu-madrid-1' + backend.namespace = 'ns' + backend.app_id = 'app-id' + backend.app_name = 'lithops_app' + backend.invoke_functions = {} + + function_ids = { + backend._format_function_name(backend._format_image_name('rt'), 256): 'fn-256', + backend._format_function_name(backend._format_image_name('rt'), 1024): 'fn-1024', + } + monkeypatch.setattr(backend, '_get_function_id', lambda name: function_ids.get(name)) + backend.cf_client = types.SimpleNamespace(get_function=lambda function_id: types.SimpleNamespace( + data=types.SimpleNamespace(lifecycle_state='ACTIVE', invoke_endpoint=f'https://{function_id}') + )) + + invoked = [] + + def invoke_client(endpoint): + def invoke_function(function_id, invoke_function_body, fn_invoke_type): + invoked.append((endpoint, function_id)) + return types.SimpleNamespace(status=202, headers={'Fn-Call-Id': f'call-{function_id}'}) + return types.SimpleNamespace(invoke_function=invoke_function) + monkeypatch.setattr(backend, '_init_functions_invk_client', invoke_client) + + backend.pre_invoke('rt', 256) + backend.pre_invoke('rt', 1024) + + # Calls of the 256MB job still queued after the 1024MB job started + assert backend.invoke('rt', 256, {}) == 'call-fn-256' + assert backend.invoke('rt', 1024, {}) == 'call-fn-1024' + + backend.invoke_functions.clear() + assert backend.invoke('rt', 256, {}) == 'call-fn-256' + assert invoked == [('https://fn-256', 'fn-256'), ('https://fn-1024', 'fn-1024'), ('https://fn-256', 'fn-256')] diff --git a/lithops/tests/test_config.py b/lithops/tests/test_config.py index 771a82e6c..36ba4d779 100644 --- a/lithops/tests/test_config.py +++ b/lithops/tests/test_config.py @@ -291,6 +291,13 @@ def test_mode_section_backend_none_is_still_applied(self): assert backend is None assert mode == c.SERVERLESS + def test_an_empty_mode_section_uses_the_default_backend(self): + # 'standalone:' with no body in the YAML file loads as None + cfg = {'lithops': {'mode': c.STANDALONE}, c.STANDALONE: None} + backend, mode = _resolve_mode_and_backend(cfg) + assert backend == c.STANDALONE_BACKEND_DEFAULT + assert mode == c.STANDALONE + def test_empty_string_backend_is_treated_as_missing(self): cfg = {'lithops': {'mode': c.LOCALHOST, 'backend': ''}} backend, mode = _resolve_mode_and_backend(cfg) diff --git a/lithops/tests/test_ibm_openwhisk_backends.py b/lithops/tests/test_ibm_openwhisk_backends.py new file mode 100644 index 000000000..b4a711fa9 --- /dev/null +++ b/lithops/tests/test_ibm_openwhisk_backends.py @@ -0,0 +1,214 @@ +# +# Unit tests for the IBM Cloud Functions, OpenWhisk, Code Engine and Singularity +# backend internals. No cloud account required; the clients are mocked. +# + +import threading +import types +from unittest.mock import MagicMock, create_autospec + +import pytest + +from lithops.serverless.backends.ibm_cf import config as cf_config +from lithops.serverless.backends.ibm_cf import ibm_cf +from lithops.serverless.backends.ibm_cf.ibm_cf import IBMCloudFunctionsBackend +from lithops.serverless.backends.openwhisk.openwhisk import OpenWhiskBackend + +THREAD_TIMEOUT = 10 + + +def _run_in_thread(target, *args): + """Runs target in a daemon thread so a deadlock or endless loop fails the test instead of hanging it""" + result = {} + + def run(): + try: + result['value'] = target(*args) + except BaseException as e: + result['error'] = e + + thread = threading.Thread(target=run, daemon=True) + thread.start() + thread.join(timeout=THREAD_TIMEOUT) + assert not thread.is_alive(), f'{target.__name__} did not return within {THREAD_TIMEOUT}s' + return result + + +def _make_cf_backend(): + backend = IBMCloudFunctionsBackend.__new__(IBMCloudFunctionsBackend) + backend.name = 'ibm_cf' + backend.config = {'runtime_timeout': 600, 'token_expiry_time': 'expiry'} + backend.is_lithops_worker = False + backend.package = 'lithops_abcd' + backend.namespace_id = 'ns-id' + backend.invoke_error = None + backend.token_manager = MagicMock() + backend.token_manager.get_token.return_value = ('token', 'expiry') + backend.cf_client = MagicMock() + return backend + + +def _make_ow_backend(): + backend = OpenWhiskBackend.__new__(OpenWhiskBackend) + backend.name = 'openwhisk' + backend.namespace = 'guest' + backend.package = 'lithops_abcde' + backend.is_lithops_worker = False + backend.cf_client = MagicMock() + return backend + + +# IBM Cloud Functions + + +def _cf_mutex_is_free(): + if not ibm_cf.cf_mutex.acquire(timeout=1): + return False + ibm_cf.cf_mutex.release() + return True + + +def test_ibm_cf_redeploy_on_404_does_not_deadlock(monkeypatch, tmp_path): + monkeypatch.setattr(cf_config, 'FH_ZIP_LOCATION', str(tmp_path / 'lithops_cf.zip')) + backend = _make_cf_backend() + backend.cf_client.invoke.side_effect = [404, 'activation-id'] + backend.cf_client.invoke_with_result.return_value = {'preinstalls': []} + + result = _run_in_thread(backend.invoke, 'lithops-cf-runtime', 256, {}) + + assert result == {'value': 'activation-id'} + backend.cf_client.create_action.assert_called_once() + assert backend.invoke_error is None + assert _run_in_thread(_cf_mutex_is_free) == {'value': True} + + +def test_ibm_cf_redeploy_failure_releases_lock(monkeypatch): + backend = _make_cf_backend() + backend.cf_client.invoke.return_value = 404 + monkeypatch.setattr(backend, 'deploy_runtime', MagicMock(side_effect=RuntimeError('deploy failed'))) + + result = _run_in_thread(backend.invoke, 'lithops-cf-runtime', 256, {}) + assert isinstance(result.get('error'), RuntimeError) + + assert _run_in_thread(_cf_mutex_is_free) == {'value': True} + + +@pytest.mark.parametrize('make_backend', [_make_cf_backend, _make_ow_backend]) +def test_runtime_meta_error_is_raised_as_exception(make_backend): + backend = make_backend() + backend.cf_client.invoke_with_result.side_effect = ValueError('connection refused') + + with pytest.raises(Exception, match='Unable to extract metadata: connection refused') as excinfo: + backend._generate_runtime_meta('lithops-runtime', 256) + + assert isinstance(excinfo.value.__cause__, ValueError) + + +# OpenWhisk + + +@pytest.mark.parametrize('status, message', [(401, 'api_key'), (404, 'not deployed')]) +def test_openwhisk_invoke_error_status_is_not_an_activation_id(status, message): + backend = _make_ow_backend() + backend.cf_client.invoke.return_value = status + + with pytest.raises(Exception, match=message): + backend.invoke('lithops-ow-runtime', 256, {}) + + +def test_openwhisk_invoke_returns_activation_id(): + backend = _make_ow_backend() + backend.cf_client.invoke.return_value = 'activation-id' + + assert backend.invoke('lithops-ow-runtime', 256, {}) == 'activation-id' + + +def test_openwhisk_invoke_throttled_returns_none(): + backend = _make_ow_backend() + backend.cf_client.invoke.return_value = None + + assert backend.invoke('lithops-ow-runtime', 256, {}) is None + + +# Code Engine + + +@pytest.fixture +def ce(): + pytest.importorskip('ibm_code_engine_sdk') + from ibm_cloud_sdk_core import ApiException + from ibm_code_engine_sdk.code_engine_v2 import CodeEngineV2 + from lithops.serverless.backends.code_engine import code_engine + + backend = code_engine.CodeEngineBackend.__new__(code_engine.CodeEngineBackend) + backend.name = 'code_engine' + backend.config = {'runtime_timeout': 600, 'runtime_cpu': 0.125} + backend.project_id = 'project-id' + backend.project_name = 'lithops-test' + backend.user_key = 'abcd' + backend.jobs = [] + backend.ce_client = create_autospec(CodeEngineV2, instance=True) + return types.SimpleNamespace(backend=backend, module=code_engine, ApiException=ApiException) + + +def test_code_engine_clear_removes_all_jobs(ce, monkeypatch): + monkeypatch.setattr(ce.backend, '_get_or_create_namespace', MagicMock(return_value='ns')) + ce.backend.jobs = ['A1-00', 'A1-01', 'A1-02', 'A1-03'] + + ce.backend.clear() + + assert ce.backend.jobs == [] + assert ce.backend.ce_client.delete_job_run.call_count == 4 + assert ce.backend.ce_client.delete_config_map.call_count == 4 + + +def test_code_engine_replace_existing_config_map(ce): + ce.backend.ce_client.create_config_map.side_effect = ce.ApiException(409, message='already exists') + + assert ce.backend._create_config_map('lithops-a1-00', {'key': 'value'}) == 'lithops-a1-00' + + ce.backend.ce_client.replace_config_map.assert_called_once() + _, kwargs = ce.backend.ce_client.replace_config_map.call_args + assert kwargs['if_match'] == '*' + assert 'lithops.payload' in kwargs['data'] + + +@pytest.mark.parametrize('stuck', ['api_error', 'pending']) +def test_code_engine_wait_for_job_run_times_out(ce, monkeypatch, stuck): + monkeypatch.setattr(ce.module.config, 'JOB_RUN_TIMEOUT', 0.2) + monkeypatch.setattr(ce.module.config, 'JOB_RUN_POLL_INTERVAL', 0.01) + if stuck == 'api_error': + ce.backend.ce_client.get_job_run.side_effect = ce.ApiException(500, message='internal error') + else: + ce.backend.ce_client.get_job_run.return_value.get_result.return_value = {'status': 'pending'} + + result = _run_in_thread(ce.backend._wait_for_job_run, 'lithops-runtime-metadata') + + assert 'Timed out' in str(result.get('error')) + + +def test_code_engine_job_definition_delete_times_out(ce, monkeypatch): + monkeypatch.setattr(ce.module.config, 'JOB_DEF_DELETE_TIMEOUT', 0.2) + + result = _run_in_thread(ce.backend._create_job_definition, 'docker.io/user/runtime:1', 256) + + assert 'Timed out' in str(result.get('error')) + ce.backend.ce_client.create_job.assert_not_called() + + +# Singularity + + +def test_singularity_runtime_meta_timeout_message(monkeypatch): + pytest.importorskip('pika') + from lithops.serverless.backends.singularity import singularity + + monkeypatch.setattr(singularity, 'time', types.SimpleNamespace(sleep=lambda seconds: None)) + backend = singularity.SingularityBackend.__new__(singularity.SingularityBackend) + backend.channel = MagicMock() + backend.internal_storage = MagicMock() + backend.internal_storage.storage.config = {} + backend.internal_storage.get_data.side_effect = Exception('NoSuchKey') + + with pytest.raises(Exception, match='Failed getting runtime metadata'): + backend._generate_runtime_meta('singularity-runtime') diff --git a/lithops/tests/test_invokers.py b/lithops/tests/test_invokers.py index 4b27da441..b54e94edd 100644 --- a/lithops/tests/test_invokers.py +++ b/lithops/tests/test_invokers.py @@ -399,7 +399,7 @@ def test_a_restart_forgets_tokens_handed_back_after_the_stop(self): patch.object(inv, '_invoke_direct'): inv.compute_handler = MagicMock() inv._invoke_job(_job(chunksize=1, total_calls=2)) - assert inv.job_monitor.token_bucket_q.qsize() == 1 + assert inv.job_monitor.token_bucket_q.empty() def test_queue_call_ranges_chunks_ids(self): inv = self._faas() @@ -584,7 +584,8 @@ def test_start_async_invokers_starts_daemon_threads(self): thread = Thread.return_value assert thread.daemon is True assert thread.start.call_count == FaaSInvoker.ASYNC_INVOKERS - assert inv.job_monitor.token_bucket_q.qsize() == FaaSInvoker.ASYNC_INVOKERS + # A token of their own would be a worker beyond max_workers + assert inv.job_monitor.token_bucket_q.empty() assert len(inv.invokers) == FaaSInvoker.ASYNC_INVOKERS def test_invoke_task_uses_data_byte_strs_when_no_data_key(self): @@ -839,6 +840,11 @@ def invoke(payload): try: inv._start_async_invokers() inv._queue_call_ranges(job, range(2)) + time.sleep(0.1) + assert invoked == [] + assert inv.pending_calls_q.qsize() == 2 + + inv.job_monitor.token_bucket_q.put('#') _wait_until(lambda: len(invoked) == 1) with lock: assert invoked == [['00000']] @@ -852,8 +858,9 @@ def invoke(payload): inv.stop() inv.executor.shutdown(wait=True) - def test_overflow_beyond_max_workers_waits_for_tokens(self): - """10 workers, 20 calls: first wave runs now, the rest wait for tokens.""" + @pytest.mark.parametrize('max_workers', [10, 1]) + def test_overflow_beyond_max_workers_waits_for_tokens(self, max_workers): + """max_workers run now, the rest of the 20 calls wait for tokens.""" invoked = [] lock = threading.Lock() @@ -862,10 +869,9 @@ def invoke(payload): invoked.append(list(payload['call_ids'])) return 'act-1' - max_workers = 10 total_calls = 20 async_invokers = FaaSInvoker.ASYNC_INVOKERS - first_wave = max_workers + async_invokers + first_wave = max_workers leftover = total_calls - first_wave inv = self._live_faas( @@ -880,10 +886,11 @@ def invoke(payload): try: inv._invoke_job(job) _wait_until(lambda: len(invoked) == first_wave) + time.sleep(0.1) assert inv.pending_calls_q.qsize() == leftover with lock: ids = [call_id for chunk in invoked for call_id in chunk] - assert len(set(ids)) == first_wave + assert len(set(ids)) == len(ids) == first_wave for _ in range(leftover): inv.job_monitor.token_bucket_q.put('#') @@ -943,7 +950,11 @@ def invoke(payload): ) try: inv._invoke_job(job) - # 2 direct workers + 1 primed async worker = 3 invokes, 6 calls + _wait_until(lambda: len(invoked) == 2) + time.sleep(0.1) + assert len(invoked) == 2 + assert inv.pending_calls_q.qsize() == 2 + inv.job_monitor.token_bucket_q.put('#') _wait_until(lambda: len(invoked) == 3) assert inv.pending_calls_q.qsize() == 1 inv.job_monitor.token_bucket_q.put('#') @@ -1065,7 +1076,6 @@ def invoke(payload): return 'act-1' inv = self._live_faas() - inv.max_workers = 0 inv.compute_handler.invoke.side_effect = invoke job = _job(total_calls=1, chunksize=1) try: diff --git a/lithops/tests/test_job.py b/lithops/tests/test_job.py index 6b3524b1d..35ad57c91 100644 --- a/lithops/tests/test_job.py +++ b/lithops/tests/test_job.py @@ -13,15 +13,18 @@ # import errno +import http.server import json import logging import os import pickle +import threading from functools import partial from types import SimpleNamespace from unittest.mock import MagicMock, patch import pytest +import requests from lithops.constants import ( LOCALHOST, @@ -833,8 +836,42 @@ def test_directory_collects_nested_py(self, tmp_path): assert all('\\' not in k for k in posix_keys) -def _head(headers): - return MagicMock(headers=headers) +def _head(headers, status_code=200): + return MagicMock(headers=headers, status_code=status_code) + + +class _UrlHandler(http.server.BaseHTTPRequestHandler): + """Serves DATA at /data, a redirect to it at /redirect and a 404 elsewhere""" + DATA = b'0123456789' * 10 + + def do_HEAD(self): + if self.path == '/redirect': + self.send_response(302) + self.send_header('Location', '/data') + self.send_header('Content-Length', '0') + elif self.path == '/data': + self.send_response(200) + self.send_header('Content-Length', str(len(self.DATA))) + self.send_header('Accept-Ranges', 'bytes') + else: + self.send_response(404) + self.send_header('Content-Length', '9') + self.end_headers() + + def log_message(self, *args): + pass + + +@pytest.fixture +def url_server(): + server = http.server.ThreadingHTTPServer(('127.0.0.1', 0), _UrlHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f'http://127.0.0.1:{server.server_address[1]}' + finally: + server.shutdown() + server.server_close() class _LogCatcher(logging.Handler): @@ -1339,6 +1376,125 @@ def test_object_storage_chunk_number_on_zero_size(self): assert parts == [] assert ppo == [0] + def test_url_head_follows_redirects(self, url_server): + """The redirect itself has no content: sized as it was, the map had no calls""" + parts, ppo = create_partitions( + {}, _storage(), [{'obj': f'{url_server}/redirect'}], None, 4, None + ) + assert ppo == [4] + assert parts[0]['obj'].total_size == len(_UrlHandler.DATA) + assert parts[0]['obj'].url == f'{url_server}/redirect' + + def test_url_error_status_raises(self, url_server): + """The error page was sized and processed as if it were the object""" + url = f'{url_server}/missing' + with pytest.raises(requests.HTTPError, match=f'{url} returned HTTP 404'): + create_partitions({}, _storage(), [{'obj': url}], None, 4, None) + + @pytest.mark.parametrize('accept_ranges', ['none', 'None', '']) + def test_url_without_byte_ranges_uses_full_object(self, accept_ranges): + """Every partition of a server that ignores ranges read the whole object""" + headers = {'content-length': '100', 'accept-ranges': accept_ranges} + with patch( + 'lithops.job.partitioner.requests.head', return_value=_head(headers) + ) as head: + parts, ppo = create_partitions( + {}, _storage(), [{'obj': 'http://example.com/a'}], 10, 4, None + ) + head.assert_called_once_with('http://example.com/a', allow_redirects=True) + assert ppo == [1] + assert parts[0]['obj'].data_byte_range is None + assert parts[0]['obj'].chunk_size == 100 + + def test_url_byte_ranges_header_is_case_insensitive(self): + headers = {'content-length': '100', 'accept-ranges': 'Bytes'} + with patch( + 'lithops.job.partitioner.requests.head', return_value=_head(headers) + ): + _, ppo = create_partitions( + {}, _storage(), [{'obj': 'http://example.com/a'}], 10, None, None + ) + assert ppo == [10] + + @pytest.mark.parametrize('size, number', [(9, 6), (100, 30), (1000, 64), (64, 64)]) + @pytest.mark.parametrize('newline', [None, '\n']) + def test_chunk_number_is_the_number_of_chunks(self, tmp_path, size, number, newline): + """It took ceil(size / number) as chunk size, which leaves fewer chunks""" + path = tmp_path / 'data.txt' + path.write_bytes(b'x' * size) + minimum = 2 if newline else 1 + expected = min(number, size // minimum) + + headers = {'content-length': str(size), 'accept-ranges': 'bytes'} + with patch( + 'lithops.job.partitioner.requests.head', return_value=_head(headers) + ): + url_parts, url_ppo = create_partitions( + {}, _storage(), [{'obj': 'http://example.com/a'}], None, number, newline + ) + internal = _storage() + internal.storage.head_object.return_value = {'content-length': str(size)} + storage_parts, storage_ppo = create_partitions( + {}, internal, [{'obj': 'localhost://bucket/a'}], None, number, newline + ) + path_parts, path_ppo = create_partitions( + {}, _storage(), [{'obj': str(path)}], None, number, newline + ) + + assert url_ppo == storage_ppo == path_ppo == [expected] + for parts in (url_parts, storage_parts): + assert [(p['obj'].data_byte_range, p['obj'].chunk_size) for p in parts] == [ + (p['obj'].data_byte_range, p['obj'].chunk_size) for p in path_parts + ] + chunk_sizes = [p['obj'].chunk_size for p in path_parts] + assert sum(chunk_sizes) == size + assert max(chunk_sizes) - min(chunk_sizes) <= 1 + assert min(chunk_sizes) >= minimum + assert [p['obj'].total_parts for p in path_parts] == [expected] * expected + + @pytest.mark.parametrize('newline', [None, '\n']) + def test_chunk_number_partitions_reassemble_the_object(self, tmp_path, newline): + from lithops.worker.jobrunner import JobRunner + + data = b''.join(b'%d\n' % i for i in range(200)) + path = tmp_path / 'data.txt' + path.write_bytes(data) + parts, ppo = create_partitions({}, _storage(), [{'obj': str(path)}], None, 7, newline) + assert ppo == [7] + + runner = JobRunner.__new__(JobRunner) + pieces = [] + for part in parts: + loaded = {'obj': part['obj']} + runner._load_object(loaded) + pieces.append(loaded['obj'].data_stream.read()) + assert b''.join(pieces) == data + + def test_chunk_number_above_the_size_makes_one_byte_chunks(self, tmp_path): + path = tmp_path / 'data.txt' + path.write_bytes(b'abc') + parts, ppo = create_partitions({}, _storage(), [{'obj': str(path)}], None, 6, None) + assert ppo == [3] + assert [p['obj'].data_byte_range for p in parts] == [(0, 0), (1, 1), (2, 2)] + + def test_missing_path_raises(self, tmp_path): + """A mistyped path was dropped, and the map ran without it""" + present = tmp_path / 'a.txt' + present.write_bytes(b'data') + missing = tmp_path / 'typo.txt' + with pytest.raises(FileNotFoundError, match='typo.txt') as excinfo: + create_partitions( + {}, _storage(), [{'obj': str(present)}, {'obj': str(missing)}], None, None, None + ) + assert excinfo.value.filename == str(missing) + + def test_paths_without_files_raise(self, tmp_path): + """As an empty bucket does, rather than a map of no calls""" + empty = tmp_path / 'empty' + (empty / 'sub').mkdir(parents=True) + with pytest.raises(Exception, match='No objects found'): + create_partitions({}, _storage(), [{'obj': str(empty)}], None, None, None) + class TestPartitionGlobsOnLocalhostStorage: diff --git a/lithops/tests/test_scripts.py b/lithops/tests/test_scripts.py index cae539b97..8ee204b15 100644 --- a/lithops/tests/test_scripts.py +++ b/lithops/tests/test_scripts.py @@ -333,6 +333,24 @@ def test_clean_local_temp_tolerates_a_missing_dir(self, tmp_path, monkeypatch): assert missing.is_dir() +class TestRuntimeUpdate: + + def test_update_caches_the_runtime_timeout_like_deploy(self): + # job.py reads runtime_timeout from the cached metadata + handler = MagicMock() + handler.get_runtime_info.return_value = {'runtime_name': 'rt', 'runtime_timeout': 600} + handler.list_runtimes.return_value = [('rt', 256, cli_module.__version__, 'w')] + handler.get_runtime_key.return_value = 'key' + handler.deploy_runtime.return_value = {'preinstalls': []} + internal_storage = MagicMock() + with patch('lithops.scripts.cli._prepare_serverless', return_value=(handler, internal_storage)): + result = _cli('runtime', 'update', 'rt') + assert result.exit_code == 0, result.output + internal_storage.put_runtime_meta.assert_called_once_with( + 'key', {'preinstalls': [], 'runtime_timeout': 600} + ) + + class TestJobWorkerList: def test_job_list_empty_does_not_crash(self): diff --git a/lithops/tests/test_standalone.py b/lithops/tests/test_standalone.py index 7e21d0a0e..059f0834b 100644 --- a/lithops/tests/test_standalone.py +++ b/lithops/tests/test_standalone.py @@ -253,6 +253,192 @@ def test_the_master_and_unrequested_workers_are_regular_vms(self, tmp_path, publ assert vm.priority is None +class TestAzureVMs: + + @pytest.fixture(autouse=True) + def _need_azure(self): + pytest.importorskip('azure.identity') + pytest.importorskip('azure.mgmt.compute') + pytest.importorskip('azure.mgmt.network') + + @staticmethod + def _config(**extra): + from lithops.standalone.backends.azure_vms.config import load_config + config_data = {'azure_vms': {'resource_group': 'rg', 'subscription_id': 's', 'region': 'westeurope'}} + config_data['azure_vms'].update(extra) + load_config(config_data) + return config_data['azure_vms'] + + def test_workers_are_not_deleted_on_dismantle_by_default(self): + # The docs and the config template say False, as True is not supported + assert self._config()['delete_on_dismantle'] is False + + @pytest.mark.parametrize('ssh_username, home_dir', [ + ('ubuntu', '/home/ubuntu'), ('azureuser', '/home/azureuser'), ('root', '/root'), + ]) + def test_the_home_dir_belongs_to_the_ssh_user(self, ssh_username, home_dir): + from lithops.standalone.backends.azure_vms.azure_vms import VMInstance + with patch.object(VMInstance, 'create_client'): + instance = VMInstance('lithops-master', self._config(ssh_username=ssh_username)) + assert instance.home_dir == home_dir + + @staticmethod + def _stoppable_vm(): + from lithops.standalone.backends.azure_vms.azure_vms import VMInstance + instance = VMInstance.__new__(VMInstance) + instance.name = 'lithops-worker-1' + instance.public = False + instance.private_ip = '10.0.0.4' + instance.delete_on_dismantle = False + instance.config = {'resource_group': 'rg'} + instance.compute_client = MagicMock() + return instance + + def test_stop_deallocates_the_vm(self): + # A VM that is only powered off is still billed + instance = self._stoppable_vm() + instance.stop() + vms = instance.compute_client.virtual_machines + vms.begin_deallocate.assert_called_once_with('rg', 'lithops-worker-1') + vms.begin_power_off.assert_not_called() + + def test_stop_inside_the_vm_powers_off_when_it_cannot_deallocate(self): + from lithops.standalone.backends.azure_vms import azure_vms + instance = self._stoppable_vm() + instance.compute_client.virtual_machines.begin_deallocate.side_effect = Exception('no credentials') + with patch.object(azure_vms.os.path, 'isfile', return_value=True), \ + patch.object(azure_vms.os, 'system') as system: + instance.stop() + system.assert_called_once_with('shutdown -h now') + + def test_stop_from_the_client_reports_a_vm_it_cannot_deallocate(self): + from lithops.standalone.backends.azure_vms import azure_vms + instance = self._stoppable_vm() + instance.compute_client.virtual_machines.begin_deallocate.side_effect = Exception('forbidden') + with patch.object(azure_vms.os.path, 'isfile', return_value=False), \ + patch.object(azure_vms.os, 'system') as system, \ + patch.object(azure_vms, 'logger') as log: + instance.stop() + system.assert_not_called() + assert 'forbidden' in log.warning.call_args[0][0] + + +class _FakeMasterSSH: + """Answers like the master service of a VM that is already set up""" + + def __init__(self, ip_address, ssh_credentials): + self.ip_address = ip_address + self.ssh_credentials = ssh_credentials + self.commands = [] + self.downloads = [] + + def run_remote_command(self, cmd, run_async=False): + self.commands.append(cmd) + if '/ping' in cmd: + return json.dumps({'response': __version__}), '' + if '/job/run' in cmd: + return json.dumps({'activationId': 'abc'}), '' + return '', '' + + def upload_local_file(self, local_src, remote_dst): + pass + + def upload_data_to_file(self, data, remote_dst): + pass + + def download_remote_file(self, remote_src, local_dst): + self.downloads.append((remote_src, local_dst)) + + def close(self): + pass + + +class TestVMBackend: + + @staticmethod + def _config(**vm_extra): + from lithops.config import default_config + vm = {'ip_address': '1.2.3.4', 'ssh_username': 'alice'} + vm.update(vm_extra) + config = default_config( + config_data={'lithops': {'backend': 'vm'}, 'vm': vm}, load_storage_config=False + ) + return config + + def _handler(self, **vm_extra): + from lithops.config import extract_standalone_config + config = self._config(**vm_extra) + handler = StandaloneHandler(extract_standalone_config(config)) + return config, handler + + @pytest.mark.parametrize('ssh_username, home_dir', [('alice', '/home/alice'), ('root', '/root')]) + def test_the_master_uses_the_documented_ssh_username(self, ssh_username, home_dir): + _, handler = self._handler(ssh_username=ssh_username) + handler.init() + master = handler.backend.master + assert master.ssh_credentials['username'] == ssh_username + assert master.home_dir == home_dir + with patch('lithops.standalone.backends.vm.vm.SSHClient', _FakeMasterSSH): + assert master.get_ssh_client().ssh_credentials['username'] == ssh_username + + def test_a_job_runs_and_is_cleared_through_the_handler(self): + config, handler = self._handler() + handler.init() + payload = { + 'executor_id': 'sess-0', 'job_id': 'M000', 'job_key': 'sess-0-M000', + 'total_calls': 2, 'worker_processes': 'AUTO', 'config': config, + } + with patch('lithops.standalone.backends.vm.vm.SSHClient', _FakeMasterSSH): + handler.invoke(payload) + ssh = handler.backend.master.get_ssh_client() + handler.clear({'sess-0-M000'}) + handler.clear() + assert payload['worker_instances'][0]['name'] == handler.backend.master.name + assert payload['worker_instances'][0]['ssh_credentials']['username'] == 'alice' + assert any('/job/run' in cmd for cmd in ssh.commands) + assert any('/job/stop' in cmd and 'sess-0-M000' in cmd for cmd in ssh.commands) + assert handler.jobs == [] + + def test_the_master_service_setup_fetches_the_key_of_the_ssh_user(self, tmp_path): + _, handler = self._handler() + handler.init() + + def handler_zip(path, files): + open(path, 'w').close() + + with patch('lithops.standalone.backends.vm.vm.SSHClient', _FakeMasterSSH), \ + patch('lithops.standalone.standalone.TEMP_DIR', str(tmp_path)), \ + patch('lithops.standalone.standalone.create_handler_zip', side_effect=handler_zip): + handler._setup_master_service() + ssh = handler.backend.master.get_ssh_client() + remote, local = ssh.downloads[0] + assert remote == '/home/alice/.ssh/lithops_id_rsa.pub' + assert local.startswith(handler.backend.cache_dir) + + def test_the_budget_keeper_gets_the_vm_it_runs_on(self): + from lithops.config import extract_standalone_config + standalone_config = extract_standalone_config(self._config()) + master_data = { + 'name': 'lithops-consume', 'instance_id': '0af1', 'private_ip': '1.2.3.4', + 'delete_on_dismantle': False, 'lithops_version': __version__, + } + keeper = BudgetKeeper(standalone_config, master_data) + assert keeper.instance.name == 'lithops-consume' + assert keeper.instance.private_ip == '1.2.3.4' + assert keeper.instance.delete_on_dismantle is False + + def test_the_master_gets_the_instance_a_job_runs_on(self): + _, handler = self._handler() + worker_info = { + 'name': 'lithops-consume', 'private_ip': '1.2.3.4', 'instance_id': '0af1', + 'ssh_credentials': {'username': 'alice'}, 'instance_type': 'VM', + } + instance = handler.backend.get_instance(**worker_info, public=False) + assert instance.name == 'lithops-consume' + assert instance.instance_type == 'VM' + assert instance.config['worker_processes'] == 'AUTO' + + class TestIBMVPCRetries: @staticmethod @@ -931,6 +1117,113 @@ def test_clear_logs_when_the_master_cannot_be_reached(self): 'Could not stop the jobs on the master:' ) + def test_clear_stops_only_the_given_jobs(self): + # get_result(fs1) must not cancel the job of fs2, still running + handler = _make_handler('create') + handler.jobs = ['sess-0-M000', 'sess-0-M001'] + handler._make_request = MagicMock(return_value=None) + handler.clear({'sess-0-M000'}) + handler._make_request.assert_called_once_with('POST', 'job/stop', ['sess-0-M000']) + assert handler.jobs == ['sess-0-M001'] + # The workers are still running the other job + handler.backend.clear.assert_not_called() + + def test_clear_stops_the_create_mode_workers_once_no_job_is_left(self): + handler = _make_handler('create') + handler.jobs = ['sess-0-M000', 'sess-0-M001'] + handler._make_request = MagicMock(return_value=None) + handler.clear({'sess-0-M000'}) + handler.clear({'sess-0-M001'}) + assert handler._make_request.call_args_list[1][0][2] == ['sess-0-M001'] + assert handler.jobs == [] + handler.backend.clear.assert_called_once_with({'sess-0-M001'}) + + def test_clear_without_job_keys_stops_every_job_and_the_workers(self): + handler = _make_handler('create') + handler.jobs = ['sess-0-M000', 'sess-0-M001'] + handler._make_request = MagicMock(return_value=None) + handler.clear() + handler._make_request.assert_called_once_with( + 'POST', 'job/stop', ['sess-0-M000', 'sess-0-M001'] + ) + assert handler.jobs == [] + handler.backend.clear.assert_called_once_with(None) + + def _master_setup_failure(self, **master_errors): + handler = self._ready_handler('create') + workers = [MagicMock(name='w0'), MagicMock(name='w1')] + handler.backend.workers = list(workers) + handler._create_workers = MagicMock(return_value=list(workers)) + handler._is_master_service_ready = MagicMock(return_value=False) + for method, error in master_errors.items(): + getattr(handler.backend.master, method).side_effect = error + return handler, workers + + @pytest.mark.parametrize('method', ['create', 'wait_ready']) + def test_invoke_stops_its_workers_when_the_master_cannot_be_set_up(self, method): + handler, workers = self._master_setup_failure(**{method: Exception('quota')}) + with pytest.raises(Exception, match='quota'): + handler.invoke(self._create_payload()) + for worker in workers: + worker.stop.assert_called_once() + assert handler.backend.workers == [] + assert handler.jobs == [] + + def test_invoke_keeps_the_master_error_when_a_worker_cannot_be_stopped(self): + handler, workers = self._master_setup_failure(create=Exception('quota')) + workers[0].stop.side_effect = Exception('stop failed') + with patch('lithops.standalone.standalone.logger') as log: + with pytest.raises(Exception, match='quota'): + handler.invoke(self._create_payload()) + workers[1].stop.assert_called_once() + assert 'stop failed' in log.error.call_args[0][0] + + def test_invoke_does_not_stop_workers_a_dismantle_already_stopped(self): + handler, workers = self._master_setup_failure() + handler._validate_master_service_setup = MagicMock() + + def dismantle_and_fail(): + handler.backend.workers = [] + raise Exception('readiness probe expired') + + handler._wait_master_service_ready = MagicMock(side_effect=dismantle_and_fail) + with pytest.raises(Exception, match='readiness probe expired'): + handler.invoke(self._create_payload()) + for worker in workers: + worker.stop.assert_not_called() + + def test_request_from_worker_sends_the_body_of_a_get(self): + handler = _make_handler() + handler.is_lithops_worker = True + payload = {'worker_instance_type': 'big', 'worker_processes': 2, 'runtime_name': 'python3'} + with patch('lithops.standalone.standalone.requests.get') as get: + get.return_value.json.return_value = [] + assert handler._make_request('GET', 'worker/get', payload) == [] + assert json.loads(get.call_args.kwargs['data']) == payload + + def test_request_from_worker_raises_on_an_error_payload(self): + handler = _make_handler() + handler.is_lithops_worker = True + with patch('lithops.standalone.standalone.requests.get') as get: + get.return_value.status_code = 404 + get.return_value.json.return_value = {'error': 'not a dictionary'} + with pytest.raises(ValueError, match='not a dictionary'): + handler._make_request('GET', 'metadata', {'runtime': 'python3'}) + + def test_request_via_ssh_raises_on_an_error_payload(self): + handler = _make_handler() + ssh = handler.backend.master.get_ssh_client.return_value + ssh.run_remote_command.return_value = ('{"error": "not a dictionary"}', '') + with pytest.raises(ValueError, match='not a dictionary'): + handler._make_request('GET', 'metadata', {'runtime': 'python3'}) + + def test_an_error_from_the_master_is_no_free_worker(self): + # The error dict used to be counted as one free worker + handler = _make_handler('reuse') + ssh = handler.backend.master.get_ssh_client.return_value + ssh.run_remote_command.return_value = ('{"error": "bad request"}', '') + assert handler._get_workers_on_master('big', 2, 'python3') == [] + class TestStandaloneRunner: diff --git a/lithops/tests/test_standalone_master.py b/lithops/tests/test_standalone_master.py index b697aeaa8..cc83313d6 100644 --- a/lithops/tests/test_standalone_master.py +++ b/lithops/tests/test_standalone_master.py @@ -444,6 +444,51 @@ def test_setup_records_why_the_installation_failed(self): ] assert any(e and 'no route' in e for e in errors) + def test_setup_does_not_stop_a_worker_it_set_up(self): + handler = self._handler() + worker = _worker_instance() + handler.backend.get_instance.return_value = worker + sa_master.redis_client.hget.return_value = WorkerStatus.STARTING.value + with patch.object(sa_master, '_worker_setup_script', return_value='SCRIPT'): + sa_master.setup_worker_create_reuse(handler, {'name': 'w'}, 'wq') + worker.stop.assert_not_called() + + def _failing_worker(self, failure): + worker = _worker_instance(config={'worker_processes': 2, 'worker_create_retries': 1}) + if failure == 'boot': + worker.wait_ready.side_effect = TimeoutError('slow') + elif failure == 'validation': + worker.validate_capabilities.side_effect = sa_master.LithopsValidationError('2 sockets') + else: + worker.get_ssh_client.return_value.upload_local_file.side_effect = OSError('no route') + return worker + + @pytest.mark.parametrize('failure, error', [ + ('boot', TimeoutError), ('validation', sa_master.LithopsValidationError), ('install', OSError), + ]) + def test_setup_stops_a_worker_it_could_not_set_up(self, failure, error): + # Nothing was installed on it, not even the budget keeper that would stop it + handler = self._handler() + worker = self._failing_worker(failure) + handler.backend.get_instance.return_value = worker + sa_master.redis_client.hget.return_value = WorkerStatus.STARTING.value + with patch.object(sa_master, '_worker_setup_script', return_value='SCRIPT'): + with pytest.raises(error): + sa_master.setup_worker_create_reuse(handler, {'name': 'w'}, 'wq') + # stop() deletes it instead when the worker is to be deleted on dismantle + worker.stop.assert_called_once() + + def test_setup_keeps_the_setup_error_when_the_worker_cannot_be_stopped(self): + handler = self._handler() + worker = self._failing_worker('boot') + worker.stop.side_effect = OSError('api down') + handler.backend.get_instance.return_value = worker + sa_master.redis_client.hget.return_value = WorkerStatus.STARTING.value + with patch.object(sa_master, 'logger') as log: + with pytest.raises(TimeoutError, match='slow'): + sa_master.setup_worker_create_reuse(handler, {'name': 'w'}, 'wq') + assert any('api down' in str(c) for c in log.error.call_args_list) + def test_consume_setup_runs_the_script_on_this_instance(self, tmp_path): handler = self._handler(exec_mode=StandaloneMode.CONSUME.value) instance = _worker_instance() From 456dca56c8bbc371c13be81684f99a5efc86a07a Mon Sep 17 00:00:00 2001 From: JosepSampe Date: Mon, 19 Oct 2026 21:26:23 +0200 Subject: [PATCH 03/13] Update openwhisk --- CHANGELOG.md | 2 +- docs/source/compute_config/openwhisk.md | 29 +- lithops/libs/openwhisk/client.py | 506 +++++++++++------- .../backends/openwhisk/openwhisk.py | 82 +-- lithops/tests/test_ibm_openwhisk_backends.py | 105 ++-- lithops/tests/test_invokers.py | 12 +- lithops/tests/test_job.py | 4 +- lithops/tests/test_openwhisk_client.py | 204 +++++++ lithops/tests/test_storage_layer.py | 4 +- lithops/tests/test_utils.py | 6 +- 10 files changed, 664 insertions(+), 290 deletions(-) create mode 100644 lithops/tests/test_openwhisk_client.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 5be0e2f61..698751f85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,7 @@ - [AWS Batch] Maps over more than 20 calls no longer fail on a fractional `max_workers`. Job definitions use the requested `runtime_memory`, checked against the Fargate combinations, and their names carry the requested version, so `lithops runtime update` and `delete` find them, also past the first page of job definitions. Deleting a runtime whose job definition is already gone works. - [IBM CF] A 404 on invoke redeploys the runtime instead of deadlocking every invoker thread. - [IBM CF] [OpenWhisk] A failure to read runtime metadata raises that error instead of a `TypeError`. -- [OpenWhisk] An invoke rejected with 401 or 404 raises instead of being reported as an activation id, which made `wait()` hang. +- [OpenWhisk] An invoke rejected with 401 or 404 raises instead of being reported as an activation id, which made `wait()` hang. The client speaks to current OpenWhisk (tested with `openwhisk/standalone:nightly`): an `http://` endpoint is used as configured, actions and packages past the first page are listed, a package that already exists is reused, deleting a missing action works, and waiting for runtime metadata follows the activation instead of invoking again for ever. A non-blocking invocation opens its own connection and closes it, so the invoker threads do not queue on the requests connection pool. - [Code Engine] `clear()` deletes every job run and config map of the executor, a leftover config map is replaced instead of failing every deploy, and waiting for a job run or a job definition deletion times out. - [Singularity] Runtime metadata that never arrives raises the intended timeout error. - [Azure Functions] An invoke that returns an HTTP error other than 429 raises. Waiting for runtime metadata with the pub/sub trigger times out after `runtime_timeout`. `runtime_memory` is honoured after the first deploy: each Flex instance memory size gets its own function app, so existing runtimes are deployed again once per size. `lithops runtime update` and `delete` find a runtime by its name. diff --git a/docs/source/compute_config/openwhisk.md b/docs/source/compute_config/openwhisk.md index 86027f501..810ba217c 100644 --- a/docs/source/compute_config/openwhisk.md +++ b/docs/source/compute_config/openwhisk.md @@ -32,6 +32,31 @@ Lithops with *OpenWhisk* as serverless compute backend. Lithops can also run fun wsk action invoke --result helloPython --param name World ``` +## Local OpenWhisk + +The current standalone build is published on Docker Hub as `openwhisk/standalone:nightly` (the same image as the `7ef091c` tag). It serves the API over plain HTTP, so the endpoint keeps the `http://` scheme: + +```bash +docker run -d --name openwhisk -h openwhisk \ + -e DOCKER_API_VERSION=1.40 \ + -p 3233:3233 -p 3232:3232 \ + -v /var/run/docker.sock:/var/run/docker.sock \ + openwhisk/standalone:nightly +``` + +The image carries Docker 18.06 to launch action containers. `DOCKER_API_VERSION=1.40` lets that client talk to a current Docker daemon, which otherwise rejects it. Its Python runtimes are `python:3.10` and `python:3.11`, and an action is limited to 128–512 MB and 300 seconds. + +The playground is at . The guest namespace and its key are printed at startup (`docker logs openwhisk | grep 'wsk property'`) and are, by default: + +```yaml +openwhisk: + endpoint : http://localhost:3233 + namespace : guest + api_key : 23bc46b1-71f6-4ed5-8c54-816aa4f8c502:123zO3xZCLrMN6v2BKK1dXYFpXlPkccOFqm12CdAsMgRU4VrNZ9lyGVCGuMDGIwP +``` + +Shut it down with `docker exec openwhisk stop`, which also removes the action containers it started. + ## Configuration 3. Edit your Lithops config and add the following keys: @@ -73,12 +98,12 @@ Lithops with *OpenWhisk* as serverless compute backend. Lithops can also run fun |openwhisk | endpoint | |yes | API Host endpoint | |openwhisk | namespace | |yes | Namespace | |openwhisk | api_key | |yes | API Auth| -|openwhisk | insecure | |yes | Insecure access | +|openwhisk | insecure | False |no | Skip TLS verification. Leave false for an `http://` endpoint | |openwhisk | max_workers | 100 | no | Max number of workers per `FunctionExecutor()`| |openwhisk | worker_processes | 1 | no | Number of Lithops processes within a given worker. This can be used to parallelize function activations within a worker | |openwhisk | runtime | |no | Docker image name | |openwhisk | runtime_memory | 256 |no | Memory limit in MB. Default 256MB | -|openwhisk | runtime_timeout | 600 |no | Runtime timeout in seconds. Default 10 minutes | +|openwhisk | runtime_timeout | 300 |no | Runtime timeout in seconds. Default 5 minutes | |openwhisk | invoke_pool_threads | 500 |no | Number of concurrent threads used for invocation | |openwhisk | runtime_include_function | False | no | If set to true, Lithops will automatically build a new runtime, including the function's code, instead of transferring it through the storage backend at invocation time. This is useful when the function's code size is large (on the order of tens of MB) and the code does not change frequently | diff --git a/lithops/libs/openwhisk/client.py b/lithops/libs/openwhisk/client.py index d1746db8d..eecc70a73 100644 --- a/lithops/libs/openwhisk/client.py +++ b/lithops/libs/openwhisk/client.py @@ -15,267 +15,411 @@ # import ssl +import time import json import base64 -import urllib3 import logging import requests +import urllib3 import http.client -from urllib.parse import urlparse +from urllib.parse import quote, urlparse from urllib3.exceptions import InsecureRequestWarning -urllib3.disable_warnings(InsecureRequestWarning) logger = logging.getLogger(__name__) +# OpenWhisk returns at most this many entities per list request. Asking for +# less silently stops at the server default, which is 30 +_LIST_LIMIT = 200 + + +def _normalize_endpoint(endpoint): + """ + Keeps the scheme the caller configured. OpenWhisk standalone serves plain + http; a host without a scheme is taken as https + """ + endpoint = endpoint.strip().rstrip('/') + if '://' not in endpoint: + endpoint = 'https://' + endpoint + return endpoint + class OpenWhiskClient: + """ + Client for the OpenWhisk REST API, api/v1, as served by OpenWhisk 2.x + (tested against the standalone build openwhisk/standalone:nightly) + """ - def __init__(self, endpoint, namespace, api_key=None, auth=None, insecure=False, user_agent=None): + def __init__(self, endpoint, namespace, api_key=None, auth=None, insecure=False, + user_agent=None, pool_maxsize=64): """ - OpenWhiskClient Constructor - - :param endpoint: OpenWhisk endpoint. + :param endpoint: OpenWhisk endpoint, including its scheme. :param namespace: User namespace. - :param api_key: User AUTH Key. HTTP Basic authentication. - :param auth: Authorization token string "Basic eyJraWQiOiIyMDE5MDcyNCIsImFsZ...". - :param insecure: Insecure backend. Disable cert verification. - :param user_agent: User agent on requests. + :param api_key: HTTP Basic credentials, "uuid:secret". + :param auth: Authorization header value, "Basic ..." or "Bearer ...". + :param insecure: Disable TLS certificate verification. + :param user_agent: Value appended to the User-Agent header. + :param pool_maxsize: Connections kept per host. The invoker calls + invoke() from a pool of this many threads. """ - self.endpoint = endpoint.replace('http:', 'https:') - self.url = f'{self.endpoint}/api/v1/namespaces' + self.endpoint = _normalize_endpoint(endpoint) self.namespace = namespace self.api_key = api_key self.auth = auth if self.api_key: - api_key = str.encode(self.api_key) - auth_token = base64.encodebytes(api_key).replace(b'\n', b'') - self.auth = 'Basic %s' % auth_token.decode('UTF-8') - - self.session = requests.session() + token = base64.b64encode(self.api_key.encode()).decode('ascii') + self.auth = f'Basic {token}' + self.session = requests.Session() + self.session.verify = not insecure if insecure: - self.session.verify = False - - self.headers = { - 'content-type': 'application/json', - 'Authorization': self.auth, - } + urllib3.disable_warnings(InsecureRequestWarning) + self.session.headers['content-type'] = 'application/json' + if self.auth: + self.session.headers['Authorization'] = self.auth if user_agent: - default_user_agent = self.session.headers['User-Agent'] - self.headers['User-Agent'] = default_user_agent + f' {user_agent}' + current = self.session.headers.get('User-Agent', 'python-requests') + self.session.headers['User-Agent'] = f'{current} {user_agent}' - self.session.headers.update(self.headers) - adapter = requests.adapters.HTTPAdapter() + adapter = requests.adapters.HTTPAdapter( + pool_connections=pool_maxsize, pool_maxsize=pool_maxsize + ) self.session.mount('https://', adapter) + self.session.mount('http://', adapter) - def create_namespace(self, namespace, resource_group_id): - """ - Create a WSK namespace - """ - data = {} + def _url(self, *parts): + path = '/'.join(quote(str(part), safe='') for part in parts) + return f'{self.endpoint}/api/v1/{path}' - if resource_group_id: - data = {"name": namespace, "description": "Auto-created Lithops namespace", - "resource_group_id": resource_group_id, "resource_plan_id": "functions-base-plan"} + def _request(self, method, url, params=None, body=None, timeout=60): + return self.session.request(method, url, params=params, json=body, timeout=timeout) - res = self.session.post(self.url, json=data) - resp_text = res.json() + @staticmethod + def _body(response): + if not response.content: + return None + try: + return response.json() + except ValueError: + return None + + def _raise(self, response, what): + body = self._body(response) + if isinstance(body, dict): + detail = body.get('error') or body.get('message') or body + else: + detail = response.text + raise Exception(f'An error occurred {what}: {detail}') + + def _list(self, url): + """Reads a whole collection. A missing one is an empty list""" + items = [] + skip = 0 + while True: + response = self._request(method='GET', url=url, params={'limit': _LIST_LIMIT, 'skip': skip}) + if response.status_code == 404: + return [] + if response.status_code != 200: + self._raise(response, f'listing {url}') + page = self._body(response) or [] + items.extend(page) + if len(page) < _LIST_LIMIT: + return items + skip += _LIST_LIMIT + def create_namespace(self, namespace, resource_group_id): + """Creates a namespace and returns its id""" + data = {} + if resource_group_id: + data = { + "name": namespace, + "description": "Auto-created Lithops namespace", + "resource_group_id": resource_group_id, + "resource_plan_id": "functions-base-plan", + } + res = self._request('POST', self._url('namespaces'), body=data) + resp_text = self._body(res) or {} if res.status_code == 201: logger.debug(f"OK --> Namespace created {namespace}") self.namespace = resp_text['id'] return resp_text['id'] - else: - msg = f"An error occurred creating the namsepace {namespace}: {resp_text['message']}" - raise Exception(msg) + msg = f"An error occurred creating the namsepace {namespace}: {resp_text.get('message')}" + raise Exception(msg) def delete_namespace(self, namespace): - """ - Delete a WSK namespace - """ - res = self.session.delete(f'{self.url}/{namespace}') - + """Deletes a namespace. A missing one is not an error""" + res = self._request('DELETE', self._url('namespaces', namespace)) if res.status_code == 200: logger.debug("OK --> Namespace deleted") elif res.status_code == 404: pass else: - resp_text = res.json() - msg = f"An error occurred deleting the namsepace: {resp_text['message']}" + resp_text = self._body(res) or {} + msg = f"An error occurred deleting the namsepace: {resp_text.get('message')}" raise Exception(msg) def list_namespaces(self, resource_group_id): - """ - List WSK namespaces - """ + """Lists namespaces. The filter is sent in the request body""" data = {} - if resource_group_id: data = {"resource_group_id": resource_group_id, "resource_plan_id": "functions-base-plan"} - - res = self.session.get(self.url, json=data) - resp_text = res.json() - + res = self._request('GET', self._url('namespaces'), body=data or None) + resp_text = self._body(res) or {} if res.status_code == 200: return resp_text - else: - msg = f"An error occurred listing the namespaces: {resp_text['message']}" - raise Exception(msg) + msg = f"An error occurred listing the namespaces: {resp_text.get('message')}" + raise Exception(msg) def create_action(self, package, action_name, image_name=None, code=None, memory=None, timeout=30000, kind='blackbox', is_binary=True, overwrite=True): """ - Create an WSK action + Creates or replaces an action. `timeout` is milliseconds, `memory` megabytes, + as the API requires. A blackbox action is a Docker image; the image may carry + its own code, or `code` is injected into it at init """ - data = {} limits = {} - cfexec = {} - limits['memory'] = memory - limits['timeout'] = timeout - data['limits'] = limits + if memory is not None: + limits['memory'] = memory + if timeout is not None: + limits['timeout'] = timeout - cfexec['kind'] = kind + action_exec = {'kind': kind} if kind == 'blackbox': - cfexec['image'] = image_name - cfexec['binary'] = is_binary - cfexec['code'] = base64.b64encode(code).decode("utf-8") if is_binary else code - data['exec'] = cfexec + if not image_name: + raise ValueError(f'Action {action_name} is a Docker action and needs an image') + action_exec['image'] = image_name + if code is not None: + action_exec['binary'] = is_binary + if is_binary: + raw = code.encode() if isinstance(code, str) else code + action_exec['code'] = base64.b64encode(raw).decode('ascii') + else: + action_exec['code'] = code - logger.debug(f'Creating function action: {action_name}') - url = '/'.join([self.url, self.namespace, 'actions', package, - action_name + "?overwrite=" + str(overwrite)]) + data = {'exec': action_exec} + if limits: + data['limits'] = limits - res = self.session.put(url, json=data) - resp_text = res.json() + logger.debug(f'Creating action: {action_name}') + url = self._url('namespaces', self.namespace, 'actions', package, action_name) + res = self._request( + 'PUT', url, params={'overwrite': 'true' if overwrite else 'false'}, body=data + ) - if res.status_code == 200: + if res.status_code in (200, 201): logger.debug(f"OK --> Created action {action_name}") - else: - msg = f'An error occurred creating/updating action {action_name}: {resp_text["error"]}' - raise Exception(msg) + return + self._raise(res, f'creating/updating action {action_name}') + + def get_action(self, package, action_name): + """Returns one action, or None when it does not exist""" + url = self._url('namespaces', self.namespace, 'actions', package, action_name) + res = self._request('GET', url) + if res.status_code == 200: + return self._body(res) + if res.status_code == 404: + return None + self._raise(res, f'getting action {action_name}') def list_actions(self, package): - """ - List all WSK actions in a package - """ + """Lists every action in a package, following the server's pages""" logger.debug(f"Listing all actions from: {package}") - url = '/'.join([self.url, self.namespace, 'actions', package, '']) - res = self.session.get(url) - if res.status_code == 200: - return res.json() - else: - return [] + # The trailing slash asks for the actions inside the package + return self._list(self._url('namespaces', self.namespace, 'actions', package, '')) def delete_action(self, package, action_name): - """ - Delete an WSK function - """ - logger.debug(f"Deleting cloud function action: {action_name}") - url = '/'.join([self.url, self.namespace, 'actions', package, action_name]) - res = self.session.delete(url) - resp_text = res.json() - - if res.status_code != 200: - logger.debug(f'An error occurred deleting action {action_name}: {resp_text["error"]}') + """Deletes an action. One that is already gone is not an error""" + logger.debug(f"Deleting action: {action_name}") + url = self._url('namespaces', self.namespace, 'actions', package, action_name) + res = self._request('DELETE', url) + if res.status_code in (200, 404): + return + self._raise(res, f'deleting action {action_name}') def list_packages(self): - """ - List all WSK packages - """ - logger.debug('Listing function packages') - url = '/'.join([self.url, self.namespace, 'packages']) - - res = self.session.get(url) - - if res.status_code == 200: - return res.json() - else: - return [] + """Lists every package in the namespace, following the server's pages""" + logger.debug('Listing packages') + return self._list(self._url('namespaces', self.namespace, 'packages')) def delete_package(self, package): - """ - Delete an WSK package - """ - logger.debug(f"Deleting functions package: {package}") - url = '/'.join([self.url, self.namespace, 'packages', package]) - res = self.session.delete(url) - resp_text = res.json() - - if res.status_code == 200: - return resp_text - else: - logger.debug(f'An error occurred deleting the package {package}: {resp_text["error"]}') + """Deletes a package. One that is already gone is not an error""" + logger.debug(f"Deleting package: {package}") + url = self._url('namespaces', self.namespace, 'packages', package) + res = self._request('DELETE', url) + if res.status_code in (200, 404): + return self._body(res) + self._raise(res, f'deleting the package {package}') def create_package(self, package): """ - Create a WSK package + Creates a package. A package that already exists is left as it is, + so a redeploy does not drop the parameters stored on it """ - logger.debug(f'Creating functions package {package}') - url = '/'.join([self.url, self.namespace, 'packages', package + "?overwrite=False"]) - - data = {"name": package} - res = self.session.put(url, json=data) - resp_text = res.json() - - if res.status_code != 200: - logger.debug(f'Package {package}: {resp_text["error"]}') - else: - logger.debug(f"OK --> Created package {package}") - - def invoke(self, package, action_name, payload={}, is_ow_action=False, self_invoked=False): + logger.debug(f'Creating package {package}') + url = self._url('namespaces', self.namespace, 'packages', package) + res = self._request( + 'PUT', url, params={'overwrite': 'false'}, body={"name": package} + ) + if res.status_code in (200, 201, 409): + logger.debug(f"OK --> Package {package} is present") + return + self._raise(res, f'creating package {package}') + + def invoke(self, package, action_name, payload=None, is_ow_action=False, self_invoked=False): """ - Invoke an WSK function by using new request. + Invokes an action without waiting for it. + + Returns the activation id, None when the namespace is throttled (429, + the caller invokes it again later), or the int 401 or 404 so the + caller can report a bad key or a missing action. Any other failure + raises. A dropped connection is tried once. + + Each call opens its own connection and closes it. A requests session + keeps a pool of 10 connections per host unless told otherwise, so the + invoker threads queue for a socket, and it sends the next call down a + socket the controller may already have closed. `is_ow_action` is + accepted for callers; a worker uses the same connection """ - url = '/'.join([self.url, self.namespace, 'actions', package, action_name]) - parsed_url = urlparse(url) - + logger.debug(f'Invoking {package}/{action_name} (worker={is_ow_action})') + url = self._url('namespaces', self.namespace, 'actions', package, action_name) try: - if is_ow_action: - resp = self.session.post(url, data=json.dumps(payload, default=str), verify=False) - resp_status = resp.status_code - data = resp.json() - else: - ctx = ssl._create_unverified_context() - conn = http.client.HTTPSConnection(parsed_url.netloc, context=ctx) - conn.request("POST", parsed_url.geturl(), - body=json.dumps(payload, default=str), - headers=self.headers) - resp = conn.getresponse() - resp_status = resp.status - data = json.loads(resp.read().decode("utf-8")) - conn.close() - except Exception as e: - logger.debug(f'Invocation Failed: {str(e)}. Doing reinvocation') - if not is_ow_action: - conn.close() + status, raw = self._post_invoke(url, payload or {}) + except (http.client.HTTPException, OSError) as e: if self_invoked: - return None - return self.invoke(package, action_name, payload, is_ow_action=is_ow_action, self_invoked=True) - - if resp_status == 202 and 'activationId' in data: - return data["activationId"] - elif resp_status == 429: - return None # "Too many concurrent requests in flight" + raise + logger.debug(f'Invocation failed: {e}. Retrying once') + return self.invoke( + package, action_name, payload, is_ow_action=is_ow_action, self_invoked=True + ) + + if status in (401, 404): + return status + data = _decode_body(raw) + if status == 202 and isinstance(data, dict) and 'activationId' in data: + return data['activationId'] + if status == 429: + return None + if isinstance(data, dict) and 'error' in data: + raise Exception(data['error']) + raise Exception(f'An error occurred invoking action {action_name}: {data or raw.decode(errors="replace")}') + + def _post_invoke(self, url, payload): + """POSTs once on a new connection and always closes it""" + parsed = urlparse(url) + body = json.dumps(payload, default=str).encode() + headers = {'Content-Type': 'application/json', 'Connection': 'close'} + authorization = self.session.headers.get('Authorization') + if authorization: + headers['Authorization'] = authorization + user_agent = self.session.headers.get('User-Agent') + if user_agent: + headers['User-Agent'] = user_agent + + if parsed.scheme == 'https': + context = ssl.create_default_context() if self.session.verify else ssl._create_unverified_context() + conn = http.client.HTTPSConnection( + parsed.hostname, parsed.port, timeout=60, context=context + ) + elif parsed.scheme == 'http': + conn = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=60) else: - if resp_status == 401: - # unauthorized. Probably token expired if using IAM auth - return resp_status - elif resp_status == 404: - # Runtime is not deployed - return resp_status - else: - logger.debug(data) - raise Exception(data['error']) + raise ValueError(f'Unsupported endpoint scheme in {url}') + + path = parsed.path or '/' + if parsed.query: + path = f'{path}?{parsed.query}' + try: + conn.request('POST', path, body=body, headers=headers) + response = conn.getresponse() + return response.status, response.read() + finally: + conn.close() + + def get_activation(self, activation_id): + """Returns an activation, or None while it is not stored yet""" + url = self._url('namespaces', self.namespace, 'activations', activation_id) + res = self._request('GET', url) + if res.status_code == 200: + return self._body(res) + if res.status_code == 404: + return None + self._raise(res, f'getting activation {activation_id}') - def invoke_with_result(self, package, action_name, payload={}): + def invoke_with_result(self, package, action_name, payload=None, timeout=180): """ - Invoke a WSK function waiting for the result. + Invokes an action and returns its result. The controller holds the + request for a short while; if the action is still running, the + activation is polled until `timeout` seconds have passed """ - url = '/'.join([self.url, self.namespace, 'actions', package, action_name + "?blocking=true&result=true"]) - resp = self.session.post(url, json=payload) - result = resp.json() - - return result + url = self._url('namespaces', self.namespace, 'actions', package, action_name) + deadline = time.monotonic() + timeout + response = None + while time.monotonic() < deadline: + remaining = max(1, deadline - time.monotonic()) + try: + # Wait out the whole budget. The controller answers earlier, with + # the result or with the activation id, and a second POST would + # start a second activation + response = self._request( + 'POST', url, + params={'blocking': 'true', 'result': 'true'}, + body=payload or {}, + timeout=(10, remaining), + ) + except requests.Timeout: + raise Exception(f'Timed out invoking {package}/{action_name}') from None + if response.status_code != 429: + break + time.sleep(1) + else: + raise Exception(f'Timed out invoking {package}/{action_name}') + + data = self._body(response) + if response.status_code == 200 and not _is_pending(data): + return data + if response.status_code in (200, 202) and _is_pending(data): + return self._wait_for_activation(data['activationId'], deadline) + self._raise(response, f'invoking action {action_name}') + + def _wait_for_activation(self, activation_id, deadline): + """Polls an activation until it has a result or the deadline passes""" + while time.monotonic() < deadline: + activation = self.get_activation(activation_id) + result = _activation_result(activation) + if result is not None: + status, value = result + if status == 'success': + return value + raise Exception(f'Activation {activation_id} failed ({status}): {value}') + time.sleep(1) + raise Exception(f'Timed out waiting for activation {activation_id}') + + +def _decode_body(raw): + if not raw: + return None + try: + return json.loads(raw) + except ValueError: + return None + + +def _is_pending(data): + """A not-yet-finished blocking call comes back as only an activation id""" + return isinstance(data, dict) and 'activationId' in data and 'preinstalls' not in data and len(data) == 1 + + +def _activation_result(activation): + """ + (status, result) once the activation has finished, None while it is running. + OpenWhisk stores an activation only after it completes, so a missing one is + still running + """ + if not isinstance(activation, dict): + return None + response = activation.get('response') + if not isinstance(response, dict) or 'status' not in response: + return None + return response['status'], response.get('result') diff --git a/lithops/serverless/backends/openwhisk/openwhisk.py b/lithops/serverless/backends/openwhisk/openwhisk.py index 01bae2a5c..8b60580dc 100644 --- a/lithops/serverless/backends/openwhisk/openwhisk.py +++ b/lithops/serverless/backends/openwhisk/openwhisk.py @@ -58,7 +58,8 @@ def __init__(self, ow_config, internal_storage): namespace=self.namespace, api_key=self.api_key, insecure=self.insecure, - user_agent=self.user_agent + user_agent=self.user_agent, + pool_maxsize=ow_config['invoke_pool_threads'], ) msg = COMPUTE_CLI_MSG.format('OpenWhisk') @@ -69,36 +70,46 @@ def _format_function_name(self, runtime_name, runtime_memory, version=__version_ return f'{runtime_name}_{runtime_memory}MB_{version}' def _unformat_function_name(self, action_name): - runtime_name, memory, version = action_name.rsplit('_', 2) - image_name_parts = runtime_name.split('_') - - domain = image_name_parts[0] - + """ + Reverses _format_function_name. The image is recovered on a best effort + basis: a registry port is the only numeric path piece, and a tag is the + piece after the image name. Raises ValueError for any other name, so + listing runtimes can skip actions that Lithops did not create + """ try: - port = int(image_name_parts[1]) - namespace = image_name_parts[2] - name = image_name_parts[3] - last_id = 4 - except Exception: - port = None - namespace = image_name_parts[1] - name = image_name_parts[2] - last_id = 3 - + runtime_name, memory, version = action_name.rsplit('_', 2) + memory_mb = int(memory[:-2]) + except ValueError: + raise ValueError(f'Not a Lithops action name: {action_name}') from None + if not memory.endswith('MB'): + raise ValueError(f'Not a Lithops action name: {action_name}') + + parts = runtime_name.split('_') + if len(parts) < 2: + raise ValueError(f'Not a Lithops action name: {action_name}') + if len(parts) == 2: + return version, f'{parts[0]}/{parts[1]}', memory_mb + + domain = parts[0] try: - tag = image_name_parts[last_id] - except Exception: - tag = None + port = int(parts[1]) + except ValueError: + port = None - if port: + if port is not None: + if len(parts) < 4: + raise ValueError(f'Not a Lithops action name: {action_name}') + namespace, name = parts[2], parts[3] + tag = parts[4] if len(parts) > 4 else None image_name = f'{domain}:{port}/{namespace}/{name}' else: + namespace, name = parts[1], parts[2] + tag = parts[3] if len(parts) > 3 else None image_name = f'{domain}/{namespace}/{name}' if tag: image_name = f'{image_name}:{tag}' - - return version, image_name, int(memory.replace('MB', '')) + return version, image_name, memory_mb def _get_default_runtime_image_name(self): try: @@ -179,12 +190,9 @@ def clean(self, **kwargs): packages = self.cf_client.list_packages() for pkg in packages: if pkg['name'].startswith('lithops') and pkg['name'].endswith(self.user_key): - actions = self.cf_client.list_actions(pkg['name']) - while actions: - for action in actions: - logger.info(f'Deleting function: {action["name"]}') - self.cf_client.delete_action(pkg['name'], action['name']) - actions = self.cf_client.list_actions(pkg['name']) + for action in self.cf_client.list_actions(pkg['name']): + logger.info(f'Deleting function: {action["name"]}') + self.cf_client.delete_action(pkg['name'], action['name']) self.cf_client.delete_package(pkg['name']) def list_runtimes(self, docker_image_name='all'): @@ -199,7 +207,11 @@ def list_runtimes(self, docker_image_name='all'): if pkg['name'] == self.package: actions = self.cf_client.list_actions(pkg['name']) for action in actions: - version, image_name, memory = self._unformat_function_name(action['name']) + try: + version, image_name, memory = self._unformat_function_name(action['name']) + except ValueError: + logger.debug(f'Skipping action {action["name"]}: not a Lithops runtime') + continue if docker_image_name == image_name or docker_image_name == 'all': runtimes.append((image_name, memory, version, action['name'])) return runtimes @@ -261,13 +273,13 @@ def _generate_runtime_meta(self, docker_image_name, memory): logger.debug(f"Extracting runtime metadata from: {docker_image_name}") action_name = self._format_function_name(docker_image_name, memory) payload = {'log_level': logger.getEffectiveLevel(), 'get_metadata': True} + # The first run also pulls the runtime image, which the action timeout + # does not cover + wait = self.ow_config.get('runtime_timeout', 300) + 180 try: - retry_invoke = True - while retry_invoke: - retry_invoke = False - runtime_meta = self.cf_client.invoke_with_result(self.package, action_name, payload) - if 'activationId' in runtime_meta: - retry_invoke = True + runtime_meta = self.cf_client.invoke_with_result( + self.package, action_name, payload, timeout=wait + ) except Exception as e: raise Exception(f"Unable to extract metadata: {e}") from e diff --git a/lithops/tests/test_ibm_openwhisk_backends.py b/lithops/tests/test_ibm_openwhisk_backends.py index b4a711fa9..2c5da7919 100644 --- a/lithops/tests/test_ibm_openwhisk_backends.py +++ b/lithops/tests/test_ibm_openwhisk_backends.py @@ -1,6 +1,6 @@ # -# Unit tests for the IBM Cloud Functions, OpenWhisk, Code Engine and Singularity -# backend internals. No cloud account required; the clients are mocked. +# Unit tests for the OpenWhisk, Code Engine and Singularity backend internals. +# No cloud account required; the clients are mocked. # import threading @@ -9,9 +9,6 @@ import pytest -from lithops.serverless.backends.ibm_cf import config as cf_config -from lithops.serverless.backends.ibm_cf import ibm_cf -from lithops.serverless.backends.ibm_cf.ibm_cf import IBMCloudFunctionsBackend from lithops.serverless.backends.openwhisk.openwhisk import OpenWhiskBackend THREAD_TIMEOUT = 10 @@ -34,68 +31,20 @@ def run(): return result -def _make_cf_backend(): - backend = IBMCloudFunctionsBackend.__new__(IBMCloudFunctionsBackend) - backend.name = 'ibm_cf' - backend.config = {'runtime_timeout': 600, 'token_expiry_time': 'expiry'} - backend.is_lithops_worker = False - backend.package = 'lithops_abcd' - backend.namespace_id = 'ns-id' - backend.invoke_error = None - backend.token_manager = MagicMock() - backend.token_manager.get_token.return_value = ('token', 'expiry') - backend.cf_client = MagicMock() - return backend - - def _make_ow_backend(): backend = OpenWhiskBackend.__new__(OpenWhiskBackend) backend.name = 'openwhisk' backend.namespace = 'guest' backend.package = 'lithops_abcde' + backend.user_key = 'abcde' backend.is_lithops_worker = False + backend.ow_config = {'runtime_timeout': 300} backend.cf_client = MagicMock() return backend -# IBM Cloud Functions - - -def _cf_mutex_is_free(): - if not ibm_cf.cf_mutex.acquire(timeout=1): - return False - ibm_cf.cf_mutex.release() - return True - - -def test_ibm_cf_redeploy_on_404_does_not_deadlock(monkeypatch, tmp_path): - monkeypatch.setattr(cf_config, 'FH_ZIP_LOCATION', str(tmp_path / 'lithops_cf.zip')) - backend = _make_cf_backend() - backend.cf_client.invoke.side_effect = [404, 'activation-id'] - backend.cf_client.invoke_with_result.return_value = {'preinstalls': []} - - result = _run_in_thread(backend.invoke, 'lithops-cf-runtime', 256, {}) - - assert result == {'value': 'activation-id'} - backend.cf_client.create_action.assert_called_once() - assert backend.invoke_error is None - assert _run_in_thread(_cf_mutex_is_free) == {'value': True} - - -def test_ibm_cf_redeploy_failure_releases_lock(monkeypatch): - backend = _make_cf_backend() - backend.cf_client.invoke.return_value = 404 - monkeypatch.setattr(backend, 'deploy_runtime', MagicMock(side_effect=RuntimeError('deploy failed'))) - - result = _run_in_thread(backend.invoke, 'lithops-cf-runtime', 256, {}) - assert isinstance(result.get('error'), RuntimeError) - - assert _run_in_thread(_cf_mutex_is_free) == {'value': True} - - -@pytest.mark.parametrize('make_backend', [_make_cf_backend, _make_ow_backend]) -def test_runtime_meta_error_is_raised_as_exception(make_backend): - backend = make_backend() +def test_runtime_meta_error_is_raised_as_exception(): + backend = _make_ow_backend() backend.cf_client.invoke_with_result.side_effect = ValueError('connection refused') with pytest.raises(Exception, match='Unable to extract metadata: connection refused') as excinfo: @@ -130,6 +79,48 @@ def test_openwhisk_invoke_throttled_returns_none(): assert backend.invoke('lithops-ow-runtime', 256, {}) is None +@pytest.mark.parametrize('image, memory', [ + ('docker.io/lithopscloud/openwhisk-python-v312:latest', 256), + ('localhost:5000/ns/img:1.2', 512), + ('user/name', 128), +]) +def test_openwhisk_action_name_round_trip(image, memory): + backend = _make_ow_backend() + name = backend._format_function_name(image, memory, '3.8.1.dev0') + assert backend._unformat_function_name(name) == ('3.8.1.dev0', image, memory) + + +def test_openwhisk_list_runtimes_skips_foreign_actions(): + backend = _make_ow_backend() + good = backend._format_function_name('docker.io/ns/rt:1', 256, '3.8.0') + backend.cf_client.list_packages.return_value = [{'name': backend.package}, {'name': 'other'}] + backend.cf_client.list_actions.return_value = [{'name': good}, {'name': 'hello'}] + + assert backend.list_runtimes() == [('docker.io/ns/rt:1', 256, '3.8.0', good)] + + +def test_openwhisk_clean_deletes_each_action_once(): + backend = _make_ow_backend() + backend.cf_client.list_packages.return_value = [{'name': backend.package}] + backend.cf_client.list_actions.return_value = [{'name': 'a'}, {'name': 'b'}] + + backend.clean() + + assert backend.cf_client.delete_action.call_count == 2 + backend.cf_client.delete_package.assert_called_once_with(backend.package) + + +def test_openwhisk_runtime_meta_waits_once(): + backend = _make_ow_backend() + backend.cf_client.invoke_with_result.return_value = {'preinstalls': ['requests']} + + assert backend._generate_runtime_meta('ns/rt:1', 256) == {'preinstalls': ['requests']} + + _, kwargs = backend.cf_client.invoke_with_result.call_args + assert kwargs['timeout'] == 480 + assert backend.cf_client.invoke_with_result.call_count == 1 + + # Code Engine diff --git a/lithops/tests/test_invokers.py b/lithops/tests/test_invokers.py index b54e94edd..93b920c5f 100644 --- a/lithops/tests/test_invokers.py +++ b/lithops/tests/test_invokers.py @@ -145,8 +145,8 @@ def _bare_invoker(**attrs): }, compute_handler=MagicMock(), internal_storage=MagicMock(), - config={'lithops': {'mode': SERVERLESS, 'backend': 'ibm_cf'}, 'ibm_cf': {}}, - backend='ibm_cf', + config={'lithops': {'mode': SERVERLESS, 'backend': 'openwhisk'}, 'openwhisk': {}}, + backend='openwhisk', include_function=False, telemetry=MagicMock(), job_monitor=MagicMock(), @@ -744,19 +744,19 @@ def _handler(self): return handler def _config(self, **backend): - ibm_cf = { + openwhisk = { 'invoke_pool_threads': 4, 'remote_invoker': False, } - ibm_cf.update(backend) + openwhisk.update(backend) return { 'lithops': { 'mode': SERVERLESS, - 'backend': 'ibm_cf', + 'backend': 'openwhisk', 'storage': 'localhost', 'telemetry': False, }, - 'ibm_cf': ibm_cf, + 'openwhisk': openwhisk, 'localhost': {'storage_bucket': 'test-bucket'}, } diff --git a/lithops/tests/test_job.py b/lithops/tests/test_job.py index 35ad57c91..1e5bc7111 100644 --- a/lithops/tests/test_job.py +++ b/lithops/tests/test_job.py @@ -242,14 +242,14 @@ def test_chunksize_overrides_config_default(self): assert job.chunksize == 7 def test_serverless_clamps_timeout(self): - cfg = _job_config(mode=SERVERLESS, backend='ibm_cf') + cfg = _job_config(mode=SERVERLESS, backend='openwhisk') job = _make_map_job(config=cfg, execution_timeout=100) assert job.runtime_memory == 256 assert job.runtime_timeout == 30 assert job.execution_timeout == 25 def test_serverless_keeps_timeout_below_runtime(self): - cfg = _job_config(mode=SERVERLESS, backend='ibm_cf') + cfg = _job_config(mode=SERVERLESS, backend='openwhisk') job = _make_map_job(config=cfg, execution_timeout=10, runtime_memory=512) assert job.runtime_memory == 512 assert job.execution_timeout == 10 diff --git a/lithops/tests/test_openwhisk_client.py b/lithops/tests/test_openwhisk_client.py new file mode 100644 index 000000000..c7f391181 --- /dev/null +++ b/lithops/tests/test_openwhisk_client.py @@ -0,0 +1,204 @@ +# +# The OpenWhisk client against a fake controller. No server is contacted. +# + +import json + +import pytest +import requests + +from lithops.libs.openwhisk.client import OpenWhiskClient + + +def _response(status, body=None): + response = requests.Response() + response.status_code = status + response.url = 'http://localhost:3233/api/v1/x' + response._content = b'' if body is None else json.dumps(body).encode() + return response + + +def _client(responses=None, errors=None): + """A client whose session replays the given responses, then the errors""" + client = OpenWhiskClient('http://localhost:3233', 'guest', api_key='uuid:secret') + calls = [] + pending = list(responses or []) + failures = list(errors or []) + + def request(method, url, params=None, json=None, timeout=None): + calls.append({'method': method, 'url': url, 'params': params, 'json': json, 'timeout': timeout}) + if failures: + raise failures.pop(0) + return pending.pop(0) + + client.session.request = request + return client, calls + + +def test_http_endpoint_is_kept_and_key_is_basic_auth(): + client, _ = _client() + assert client.endpoint == 'http://localhost:3233' + assert client.session.headers['Authorization'].startswith('Basic ') + + bare = OpenWhiskClient('ow.example.com', 'guest', auth='Bearer token') + assert bare.endpoint == 'https://ow.example.com' + assert bare.session.headers['Authorization'] == 'Bearer token' + + +def test_create_action_sends_docker_exec(): + client, calls = _client([_response(200, {})]) + client.create_action('pkg', 'hello', image_name='openwhisk/python3action', code=b'zip', memory=256, timeout=60000) + + call = calls[0] + assert call['method'] == 'PUT' + assert call['url'] == 'http://localhost:3233/api/v1/namespaces/guest/actions/pkg/hello' + assert call['params'] == {'overwrite': 'true'} + assert call['json']['limits'] == {'memory': 256, 'timeout': 60000} + assert call['json']['exec']['kind'] == 'blackbox' + assert call['json']['exec']['image'] == 'openwhisk/python3action' + assert call['json']['exec']['binary'] is True + assert call['json']['exec']['code'] # base64 of the zip + + +def test_source_action_has_no_image(): + client, calls = _client([_response(200, {})]) + client.create_action('pkg', 'hello', code='def main(args):\n return {}\n', kind='python:3', is_binary=False) + exec_ = calls[0]['json']['exec'] + assert exec_ == {'kind': 'python:3', 'binary': False, 'code': 'def main(args):\n return {}\n'} + + +def test_docker_action_without_image_is_rejected(): + client, calls = _client() + with pytest.raises(ValueError, match='image'): + client.create_action('pkg', 'hello', kind='blackbox') + assert calls == [] + + +def test_lists_follow_pages_and_a_missing_collection_is_empty(): + page = [{'name': str(i)} for i in range(200)] + client, calls = _client([_response(200, page), _response(200, [{'name': 'last'}]), _response(404)]) + + assert len(client.list_actions('pkg')) == 201 + assert [call['params']['skip'] for call in calls[:2]] == [0, 200] + assert calls[0]['url'].endswith('/actions/pkg/') + assert client.list_packages() == [] + + +def test_delete_of_missing_action_and_existing_package(): + client, calls = _client([_response(404), _response(409)]) + client.delete_action('pkg', 'gone') + client.create_package('pkg') + assert calls[1]['params'] == {'overwrite': 'false'} + + +def _scripted_invoke(client, outcomes): + """Makes invoke() replay (status, body) pairs, or raise the given errors""" + calls = [] + pending = list(outcomes) + + def post(url, payload): + calls.append({'url': url, 'payload': payload}) + outcome = pending.pop(0) + if isinstance(outcome, Exception): + raise outcome + status, body = outcome + raw = b'' if body is None else json.dumps(body).encode() + return status, raw + + client._post_invoke = post + return calls + + +def test_invoke_returns_activation_id_status_or_none(): + client, _ = _client() + calls = _scripted_invoke(client, [ + (202, {'activationId': 'act-1'}), + (401, None), + (404, None), + (429, None), + (400, {'error': 'The requested action does not exist'}), + ]) + assert client.invoke('pkg', 'hello', {'n': 1}) == 'act-1' + assert calls[0]['payload'] == {'n': 1} + assert calls[0]['url'].startswith('http://localhost:3233/') + assert client.invoke('pkg', 'hello') == 401 + assert client.invoke('pkg', 'hello') == 404 + assert client.invoke('pkg', 'hello') is None + with pytest.raises(Exception, match='does not exist'): + client.invoke('pkg', 'hello') + + +def test_invoke_retries_a_dropped_connection_once(): + client, _ = _client() + calls = _scripted_invoke(client, [OSError('reset'), (202, {'activationId': 'act-1'})]) + assert client.invoke('pkg', 'hello') == 'act-1' + assert len(calls) == 2 + + client, _ = _client() + calls = _scripted_invoke(client, [OSError('reset'), OSError('reset')]) + with pytest.raises(OSError): + client.invoke('pkg', 'hello') + assert len(calls) == 2 + + +def test_invoke_opens_a_fresh_connection_and_closes_it(monkeypatch): + opened = [] + + class _Response: + status = 202 + + @staticmethod + def read(): + return b'{"activationId": "act-1"}' + + class _Conn: + def __init__(self, host, port=None, timeout=None, context=None): + opened.append(self) + self.host, self.port, self.timeout, self.context = host, port, timeout, context + self.closed = False + + def request(self, method, path, body=None, headers=None): + self.sent = (method, path, body, headers) + + def getresponse(self): + return _Response() + + def close(self): + self.closed = True + + monkeypatch.setattr('lithops.libs.openwhisk.client.http.client.HTTPConnection', _Conn) + client, _ = _client() + assert client.invoke('pkg', 'hello', {'n': 1}) == 'act-1' + conn = opened[0] + assert conn.closed + assert conn.host == 'localhost' and conn.port == 3233 + method, path, body, headers = conn.sent + assert method == 'POST' + assert path == '/api/v1/namespaces/guest/actions/pkg/hello' + assert headers['Connection'] == 'close' + assert json.loads(body) == {'n': 1} + + monkeypatch.setattr('lithops.libs.openwhisk.client.http.client.HTTPSConnection', _Conn) + secure = OpenWhiskClient('https://ow.example.com', 'guest', api_key='uuid:secret', insecure=True) + assert secure.invoke('pkg', 'hello') == 'act-1' + assert opened[1].context is not None + + +def test_invoke_with_result_polls_until_the_activation_finishes(monkeypatch): + monkeypatch.setattr('lithops.libs.openwhisk.client.time.sleep', lambda seconds: None) + client, _ = _client([ + _response(202, {'activationId': 'act-1'}), + _response(404), + _response(200, {'activationId': 'act-1', 'response': {'status': 'success', 'result': {'ok': True}}}), + ]) + assert client.invoke_with_result('pkg', 'hello', {'get_metadata': True}) == {'ok': True} + + +def test_invoke_with_result_reports_a_failed_activation(monkeypatch): + monkeypatch.setattr('lithops.libs.openwhisk.client.time.sleep', lambda seconds: None) + client, _ = _client([ + _response(200, {'activationId': 'act-1'}), + _response(200, {'response': {'status': 'application error', 'result': {'error': 'boom'}}}), + ]) + with pytest.raises(Exception, match='boom'): + client.invoke_with_result('pkg', 'hello', timeout=5) diff --git a/lithops/tests/test_storage_layer.py b/lithops/tests/test_storage_layer.py index be4a9cf44..cc7271ede 100644 --- a/lithops/tests/test_storage_layer.py +++ b/lithops/tests/test_storage_layer.py @@ -1235,7 +1235,6 @@ def _load(backend, **ibm_cos): return config['ibm_cos'] @pytest.mark.parametrize('backend, private_endpoint', [ - ('ibm_cf', 'https://s3.private.eu-de.cloud-object-storage.appdomain.cloud'), ('code_engine', 'https://s3.direct.eu-de.cloud-object-storage.appdomain.cloud'), ('ibm_vpc', 'https://s3.direct.eu-de.cloud-object-storage.appdomain.cloud'), ('localhost', None), @@ -1246,8 +1245,7 @@ def test_region_sets_the_endpoints_of_the_compute_backend(self, backend, private assert cos.get('private_endpoint') == private_endpoint @pytest.mark.parametrize('backend, private_endpoint, error', [ - ('ibm_cf', None, 'You must provide the private_endpoint'), - ('ibm_cf', 'https://s3.direct.x', 'is not valid$'), + ('code_engine', None, 'You must provide the private_endpoint'), ('code_engine', 'https://s3.private.x', 'is not valid$'), ('ibm_vpc', 'https://s3.private.x', 'is not valid. You must use the "direct" endpoint$'), ('ibm_vpc', 's3.direct.x', 'Private Endpoint must start with http'), diff --git a/lithops/tests/test_utils.py b/lithops/tests/test_utils.py index 08717e1a2..476b36821 100644 --- a/lithops/tests/test_utils.py +++ b/lithops/tests/test_utils.py @@ -577,10 +577,10 @@ def test_docker_login_requires_credentials(self): def test_get_default_container_name_variants(self): cfg = {'docker_server': 'docker.io', 'docker_user': 'alice'} - name = get_default_container_name('ibm_cf', cfg, 'lithops') + name = get_default_container_name('openwhisk', cfg, 'lithops') assert name.startswith('docker.io/alice/lithops-v') cfg = {'docker_server': 'icr.io', 'docker_namespace': 'ns'} - name = get_default_container_name('ibm_cf', cfg, 'lithops') + name = get_default_container_name('openwhisk', cfg, 'lithops') assert name.startswith('icr.io/ns/lithops-v') cfg = { 'docker_server': 'us-docker.pkg.dev', @@ -595,7 +595,7 @@ def test_get_default_container_name_variants(self): def test_get_default_container_name_missing_docker_user(self): with pytest.raises(Exception, match='docker_user'): - get_default_container_name('ibm_cf', {'docker_server': 'docker.io'}, 'r') + get_default_container_name('openwhisk', {'docker_server': 'docker.io'}, 'r') def test_is_podman(self, monkeypatch): monkeypatch.setattr( From 32e171825785e71a1d4426abf63d8ec118ee95af Mon Sep 17 00:00:00 2001 From: JosepSampe Date: Mon, 19 Oct 2026 22:32:49 +0200 Subject: [PATCH 04/13] Fix openwhisk --- CHANGELOG.md | 6 ++---- docs/source/compute_config/openwhisk.md | 4 ++-- lithops/serverless/backends/openwhisk/config.py | 6 ++++-- lithops/standalone/standalone.py | 11 +++++++++-- lithops/tests/test_standalone.py | 15 +++++++++++++++ runtime/openwhisk/Dockerfile | 10 ++++------ runtime/openwhisk/README.md | 6 ++++-- 7 files changed, 40 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 698751f85..b6065a44f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,12 +24,10 @@ - [Core] An executor that fails to be created no longer logs a misleading `job_monitor` error at exit. - [Core] The FaaS invoker no longer runs up to two workers beyond `max_workers`. `obj_chunk_number` splits an object into exactly that many chunks, or fewer non-empty ones when it is smaller. URL inputs are sized following redirects, an HTTP error raises, and only `Accept-Ranges: bytes` enables partitioning. A local input path that does not exist raises `FileNotFoundError`, and local inputs with no files raise like an empty bucket. A mode section with no body in the config file no longer crashes config loading. - [CLI] `lithops runtime update` stores `runtime_timeout` in the runtime metadata, which every later job needed. -- [Standalone] `get_result()` on one job no longer cancels the other jobs of the executor, and create mode stops its workers only once none of its jobs is left. Worker VMs whose setup fails, and the workers of a job whose master cannot be set up, are stopped instead of left running. Requests to the master from inside a worker send their body, and an error from the master raises. The `vm` backend runs jobs: it reads `ssh_username` and provides what the master and its budget keeper need. Azure VMs are deallocated when stopped, so they are no longer billed, the home directory follows `ssh_username`, and `delete_on_dismantle` defaults to `False` as documented. +- [Standalone] `get_result()` on one job no longer cancels the other jobs of the executor, and create mode stops its workers only once none of its jobs is left. Worker VMs whose setup fails, and the workers of a job whose master cannot be set up, are stopped instead of left running. Requests to the master from inside a worker send their body, and an error from the master raises. They wait up to 60 seconds for an answer instead of 1, and as long as the master needs to pull the runtime for its metadata; a POST no longer waits for ever. The `vm` backend runs jobs: it reads `ssh_username` and provides what the master and its budget keeper need. Azure VMs are deallocated when stopped, so they are no longer billed, the home directory follows `ssh_username`, and `delete_on_dismantle` defaults to `False` as documented. - [AWS Lambda] Invocations retry connection errors a few times, re-signing each attempt, instead of retrying any error for ever. `lithops runtime update` updates the code, configuration and tags of a deployed function, and the default runtime gets the `user_tags` too. `env_vars` and `user_tags` accept a `{NAME: VALUE}` mapping or a list of `{name, value}`. Layers with more than 50 versions are deleted. Runtimes and layers past the first page of the listing are found by `lithops runtime list` and `clean`, and deleting a runtime whose function is already gone works. - [AWS Batch] Maps over more than 20 calls no longer fail on a fractional `max_workers`. Job definitions use the requested `runtime_memory`, checked against the Fargate combinations, and their names carry the requested version, so `lithops runtime update` and `delete` find them, also past the first page of job definitions. Deleting a runtime whose job definition is already gone works. -- [IBM CF] A 404 on invoke redeploys the runtime instead of deadlocking every invoker thread. -- [IBM CF] [OpenWhisk] A failure to read runtime metadata raises that error instead of a `TypeError`. -- [OpenWhisk] An invoke rejected with 401 or 404 raises instead of being reported as an activation id, which made `wait()` hang. The client speaks to current OpenWhisk (tested with `openwhisk/standalone:nightly`): an `http://` endpoint is used as configured, actions and packages past the first page are listed, a package that already exists is reused, deleting a missing action works, and waiting for runtime metadata follows the activation instead of invoking again for ever. A non-blocking invocation opens its own connection and closes it, so the invoker threads do not queue on the requests connection pool. +- [OpenWhisk] An invoke rejected with 401 or 404 raises instead of being reported as an activation id, which made `wait()` hang. A failure to read runtime metadata raises that error instead of a `TypeError`. Python 3.13 and 3.14 clients get a default runtime, built from the runtime Dockerfile with `--build-arg PYTHON_VERSION`. The client speaks to current OpenWhisk (tested with `openwhisk/standalone:nightly`): an `http://` endpoint is used as configured, actions and packages past the first page are listed, a package that already exists is reused, deleting a missing action works, and waiting for runtime metadata follows the activation instead of invoking again for ever. A non-blocking invocation opens its own connection and closes it, so the invoker threads do not queue on the requests connection pool. - [Code Engine] `clear()` deletes every job run and config map of the executor, a leftover config map is replaced instead of failing every deploy, and waiting for a job run or a job definition deletion times out. - [Singularity] Runtime metadata that never arrives raises the intended timeout error. - [Azure Functions] An invoke that returns an HTTP error other than 429 raises. Waiting for runtime metadata with the pub/sub trigger times out after `runtime_timeout`. `runtime_memory` is honoured after the first deploy: each Flex instance memory size gets its own function app, so existing runtimes are deployed again once per size. `lithops runtime update` and `delete` find a runtime by its name. diff --git a/docs/source/compute_config/openwhisk.md b/docs/source/compute_config/openwhisk.md index 810ba217c..96a3296aa 100644 --- a/docs/source/compute_config/openwhisk.md +++ b/docs/source/compute_config/openwhisk.md @@ -99,12 +99,12 @@ Shut it down with `docker exec openwhisk stop`, which also removes the action co |openwhisk | namespace | |yes | Namespace | |openwhisk | api_key | |yes | API Auth| |openwhisk | insecure | False |no | Skip TLS verification. Leave false for an `http://` endpoint | -|openwhisk | max_workers | 100 | no | Max number of workers per `FunctionExecutor()`| +|openwhisk | max_workers | 100 | no | Max number of workers per `FunctionExecutor()`. Keep it within the concurrent invocations the namespace allows: the invocations over that limit are rejected and retried | |openwhisk | worker_processes | 1 | no | Number of Lithops processes within a given worker. This can be used to parallelize function activations within a worker | |openwhisk | runtime | |no | Docker image name | |openwhisk | runtime_memory | 256 |no | Memory limit in MB. Default 256MB | |openwhisk | runtime_timeout | 300 |no | Runtime timeout in seconds. Default 5 minutes | -|openwhisk | invoke_pool_threads | 500 |no | Number of concurrent threads used for invocation | +|openwhisk | invoke_pool_threads | 100 |no | Number of concurrent threads used for invocation | |openwhisk | runtime_include_function | False | no | If set to true, Lithops will automatically build a new runtime, including the function's code, instead of transferring it through the storage backend at invocation time. This is useful when the function's code size is large (on the order of tens of MB) and the code does not change frequently | ## Test Lithops diff --git a/lithops/serverless/backends/openwhisk/config.py b/lithops/serverless/backends/openwhisk/config.py index a7cad3cda..59cb7549a 100644 --- a/lithops/serverless/backends/openwhisk/config.py +++ b/lithops/serverless/backends/openwhisk/config.py @@ -19,7 +19,9 @@ AVAILABLE_PY_RUNTIMES = { '3.10': 'docker.io/lithopscloud/openwhisk-python-v310', '3.11': 'docker.io/lithopscloud/openwhisk-python-v311', - '3.12': 'docker.io/lithopscloud/openwhisk-python-v312' + '3.12': 'docker.io/lithopscloud/openwhisk-python-v312', + '3.13': 'docker.io/lithopscloud/openwhisk-python-v313', + '3.14': 'docker.io/lithopscloud/openwhisk-python-v314', } DEFAULT_CONFIG_KEYS = { @@ -27,7 +29,7 @@ 'runtime_memory': 256, # Default memory: 256 MB 'max_workers': 100, 'worker_processes': 1, - 'invoke_pool_threads': 500, + 'invoke_pool_threads': 100, 'docker_server': 'docker.io' } diff --git a/lithops/standalone/standalone.py b/lithops/standalone/standalone.py index e57f44fac..cae71c2c1 100644 --- a/lithops/standalone/standalone.py +++ b/lithops/standalone/standalone.py @@ -51,6 +51,11 @@ _CURL_BODY_INLINE_LIMIT = 130000 +_WORKER_CONNECT_TIMEOUT = 10 +_WORKER_READ_TIMEOUT = 60 +# The master pulls and runs the runtime image before it answers +_WORKER_SLOW_ENDPOINTS = {'metadata'} + class StandaloneHandler: """ @@ -127,10 +132,12 @@ def _make_request(self, method: str, endpoint: str, data=None): def _request_from_worker(self, method: str, endpoint: str, data=None): """Calls the master service over HTTP, from inside the network""" url = self._master_url(endpoint) + read_timeout = None if endpoint in _WORKER_SLOW_ENDPOINTS else _WORKER_READ_TIMEOUT + timeout = (_WORKER_CONNECT_TIMEOUT, read_timeout) if method == 'GET': - resp = requests.get(url, data=json.dumps(data), timeout=1) + resp = requests.get(url, data=json.dumps(data), timeout=timeout) elif method == 'POST': - resp = requests.post(url, data=json.dumps(data)) + resp = requests.post(url, data=json.dumps(data), timeout=timeout) else: raise ValueError(f'Unsupported HTTP method: {method}') if not resp.content: diff --git a/lithops/tests/test_standalone.py b/lithops/tests/test_standalone.py index 059f0834b..08fc8a10c 100644 --- a/lithops/tests/test_standalone.py +++ b/lithops/tests/test_standalone.py @@ -1057,6 +1057,21 @@ def test_request_from_worker_uses_lithops_master_host(self): } assert 'lithops-master' in get.call_args[0][0] + def test_request_from_worker_gives_the_master_time_to_answer(self): + handler = _make_handler() + handler.is_lithops_worker = True + with patch('lithops.standalone.standalone.requests.get') as get: + get.return_value.json.return_value = [] + handler._make_request('GET', 'worker/list') + assert get.call_args.kwargs['timeout'] == (10, 60) + get.return_value.json.return_value = {'preinstalls': []} + handler._make_request('GET', 'metadata', {'runtime': 'img'}) + assert get.call_args.kwargs['timeout'] == (10, None) + with patch('lithops.standalone.standalone.requests.post') as post: + post.return_value.content = b'' + handler._make_request('POST', 'clean') + assert post.call_args.kwargs['timeout'] == (10, 60) + def test_request_from_worker_accepts_an_empty_post_body(self): handler = _make_handler() handler.is_lithops_worker = True diff --git a/runtime/openwhisk/Dockerfile b/runtime/openwhisk/Dockerfile index 606d1e26c..64840079b 100644 --- a/runtime/openwhisk/Dockerfile +++ b/runtime/openwhisk/Dockerfile @@ -1,8 +1,6 @@ -# Python 3.10 -#FROM python:3.10-slim-bookworm - -# Python 3.11 -FROM python:3.11-slim-bookworm +# Python 3.10 to 3.14, for example: --build-arg PYTHON_VERSION=3.13 +ARG PYTHON_VERSION=3.11 +FROM python:${PYTHON_VERSION}-slim-bookworm ENV FLASK_PROXY_PORT=8080 @@ -16,7 +14,7 @@ RUN apt-get update \ && apt-cache search linux-headers-generic COPY requirements.txt requirements.txt -RUN pip install --upgrade pip setuptools six && pip install --no-cache-dir -r requirements.txt +RUN pip install --upgrade pip setuptools six && pip install --no-cache-dir --prefer-binary -r requirements.txt # create action working directory RUN mkdir -p /action \ diff --git a/runtime/openwhisk/README.md b/runtime/openwhisk/README.md index 19e3aa940..498a4e3b3 100644 --- a/runtime/openwhisk/README.md +++ b/runtime/openwhisk/README.md @@ -46,11 +46,13 @@ fexec = lithops.FunctionExecutor(runtime_memory=512) $ docker login Navigate to [openwhisk/](openwhisk/) and update the Dockerfile that better fits to your requirements with your required system packages and Python modules. - If you need another Python version, for example Python v3.12, you must use this [Dockerfile](openwhisk/Dockerfile) that - points to a source image based on Python 3.12. Finally run the build script: + The [Dockerfile](Dockerfile) builds on Python 3.11. For another Python version, for example Python 3.13, pass it as a + build argument. Finally run the build script: $ lithops runtime build -b openwhisk docker_username/runtimename:tag + $ lithops runtime build -b openwhisk docker_username/runtimename:tag --build-arg PYTHON_VERSION=3.13 + Note that Docker Hub image names look like *"docker_username/runtimename:tag"* and must be all lower case, for example: $ lithops runtime build -b openwhisk jsampe/lithops-custom-runtime-v312:0.1 From 79192fd6881232fa83d4ebd1c88ebaee48824ca6 Mon Sep 17 00:00:00 2001 From: JosepSampe Date: Mon, 19 Oct 2026 23:03:35 +0200 Subject: [PATCH 05/13] fix backends --- CHANGELOG.md | 13 +- docs/source/compute_config/openwhisk.md | 18 ++ lithops/executors.py | 3 + .../backends/aws_batch/aws_batch.py | 161 ++++++++++-------- .../backends/code_engine/code_engine.py | 22 ++- .../backends/gcp_cloudrun/cloudrun.py | 36 ++-- .../backends/gcp_functions/gcp_functions.py | 41 +++-- .../backends/openwhisk/openwhisk.py | 4 +- .../serverless/backends/oracle_f/oracle_f.py | 21 ++- .../standalone/backends/aws_ec2/aws_ec2.py | 31 ++-- lithops/tests/test_aws_backends.py | 39 +++++ .../test_azure_aliyun_oracle_backends.py | 34 ++++ lithops/tests/test_executors.py | 12 ++ lithops/tests/test_ibm_openwhisk_backends.py | 22 +++ lithops/tests/test_serverless.py | 91 ++++++++++ lithops/tests/test_standalone.py | 29 ++++ runtime/openwhisk/README.md | 2 + 17 files changed, 437 insertions(+), 142 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6065a44f..b2151c695 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,18 +22,19 @@ - [Monitoring] Worker tokens of jobs dropped by `wait()`, replaced monitors and restarted invokers are handed back once, a held `__end__` is no longer replaced by an `__init__`, and the workers of a job that owes no token are forgotten instead of kept for as long as the job monitor runs; the shared RabbitMQ connection of a worker is closed at exit. - [Concurrent] `wait()` and `as_completed()` accept generators. - [Core] An executor that fails to be created no longer logs a misleading `job_monitor` error at exit. -- [Core] The FaaS invoker no longer runs up to two workers beyond `max_workers`. `obj_chunk_number` splits an object into exactly that many chunks, or fewer non-empty ones when it is smaller. URL inputs are sized following redirects, an HTTP error raises, and only `Accept-Ranges: bytes` enables partitioning. A local input path that does not exist raises `FileNotFoundError`, and local inputs with no files raise like an empty bucket. A mode section with no body in the config file no longer crashes config loading. +- [Core] The FaaS invoker no longer runs up to two workers beyond `max_workers`. A `wait()` that times out drops the invocations still queued, the same as any other failed wait. `obj_chunk_number` splits an object into exactly that many chunks, or fewer non-empty ones when it is smaller. URL inputs are sized following redirects, an HTTP error raises, and only `Accept-Ranges: bytes` enables partitioning. A local input path that does not exist raises `FileNotFoundError`, and local inputs with no files raise like an empty bucket. A mode section with no body in the config file no longer crashes config loading. - [CLI] `lithops runtime update` stores `runtime_timeout` in the runtime metadata, which every later job needed. -- [Standalone] `get_result()` on one job no longer cancels the other jobs of the executor, and create mode stops its workers only once none of its jobs is left. Worker VMs whose setup fails, and the workers of a job whose master cannot be set up, are stopped instead of left running. Requests to the master from inside a worker send their body, and an error from the master raises. They wait up to 60 seconds for an answer instead of 1, and as long as the master needs to pull the runtime for its metadata; a POST no longer waits for ever. The `vm` backend runs jobs: it reads `ssh_username` and provides what the master and its budget keeper need. Azure VMs are deallocated when stopped, so they are no longer billed, the home directory follows `ssh_username`, and `delete_on_dismantle` defaults to `False` as documented. +- [Standalone] Cleaning AWS EC2 workers reads every page of instances, so workers past the first 1,000 are stopped too. `get_result()` on one job no longer cancels the other jobs of the executor, and create mode stops its workers only once none of its jobs is left. Worker VMs whose setup fails, and the workers of a job whose master cannot be set up, are stopped instead of left running. Requests to the master from inside a worker send their body, and an error from the master raises. They wait up to 60 seconds for an answer instead of 1, and as long as the master needs to pull the runtime for its metadata; a POST no longer waits for ever. The `vm` backend runs jobs: it reads `ssh_username` and provides what the master and its budget keeper need. Azure VMs are deallocated when stopped, so they are no longer billed, the home directory follows `ssh_username`, and `delete_on_dismantle` defaults to `False` as documented. - [AWS Lambda] Invocations retry connection errors a few times, re-signing each attempt, instead of retrying any error for ever. `lithops runtime update` updates the code, configuration and tags of a deployed function, and the default runtime gets the `user_tags` too. `env_vars` and `user_tags` accept a `{NAME: VALUE}` mapping or a list of `{name, value}`. Layers with more than 50 versions are deleted. Runtimes and layers past the first page of the listing are found by `lithops runtime list` and `clean`, and deleting a runtime whose function is already gone works. -- [AWS Batch] Maps over more than 20 calls no longer fail on a fractional `max_workers`. Job definitions use the requested `runtime_memory`, checked against the Fargate combinations, and their names carry the requested version, so `lithops runtime update` and `delete` find them, also past the first page of job definitions. Deleting a runtime whose job definition is already gone works. -- [OpenWhisk] An invoke rejected with 401 or 404 raises instead of being reported as an activation id, which made `wait()` hang. A failure to read runtime metadata raises that error instead of a `TypeError`. Python 3.13 and 3.14 clients get a default runtime, built from the runtime Dockerfile with `--build-arg PYTHON_VERSION`. The client speaks to current OpenWhisk (tested with `openwhisk/standalone:nightly`): an `http://` endpoint is used as configured, actions and packages past the first page are listed, a package that already exists is reused, deleting a missing action works, and waiting for runtime metadata follows the activation instead of invoking again for ever. A non-blocking invocation opens its own connection and closes it, so the invoker threads do not queue on the requests connection pool. -- [Code Engine] `clear()` deletes every job run and config map of the executor, a leftover config map is replaced instead of failing every deploy, and waiting for a job run or a job definition deletion times out. +- [AWS Batch] Maps over more than 20 calls no longer fail on a fractional `max_workers`. Job definitions use the requested `runtime_memory`, checked against the Fargate combinations, and their names carry the requested version, so `lithops runtime update` and `delete` find them, also past the first page of job definitions. Deleting a runtime whose job definition is already gone works. Job queues and compute environments past the first page are found, and `lithops runtime update` registers a new job definition revision instead of keeping the CPU, roles and image from the first deploy. +- [OpenWhisk] An invoke rejected with 401 or 404 raises instead of being reported as an activation id, which made `wait()` hang. A failure to read runtime metadata raises that error instead of a `TypeError`. Python 3.13 and 3.14 clients get a default runtime, built from the runtime Dockerfile with `--build-arg PYTHON_VERSION`. The client speaks to current OpenWhisk (tested with `openwhisk/standalone:nightly`): an `http://` endpoint is used as configured, actions and packages past the first page are listed, a package that already exists is reused, deleting a missing action works, and waiting for runtime metadata follows the activation instead of invoking again for ever. A non-blocking invocation opens its own connection and closes it, so the invoker threads do not queue on the requests connection pool. `lithops runtime build` builds for the machine it runs on; `--platform linux/amd64` selects amd64. +- [Code Engine] Projects, job definitions and config maps past the first page are listed. `clear()` deletes every job run and config map of the executor, a leftover config map is replaced instead of failing every deploy, and waiting for a job run or a job definition deletion times out. - [Singularity] Runtime metadata that never arrives raises the intended timeout error. - [Azure Functions] An invoke that returns an HTTP error other than 429 raises. Waiting for runtime metadata with the pub/sub trigger times out after `runtime_timeout`. `runtime_memory` is honoured after the first deploy: each Flex instance memory size gets its own function app, so existing runtimes are deployed again once per size. `lithops runtime update` and `delete` find a runtime by its name. - [Azure Container Apps] `lithops runtime update` and `delete` find a runtime by its image, and `list_runtimes` reports memory as a number. - [Aliyun FC] Retries while reading runtime metadata resend the payload instead of an empty body. -- [Oracle Functions] Calls run on the function of their own runtime and memory, not on the function of the job started last. +- [Oracle Functions] Calls run on the function of their own runtime and memory, not on the function of the job started last. Functions and applications past the first page are found by `lithops runtime list`, `delete` and `clean`. +- [GCP Functions] [GCP Cloud Run] An HTTPS invocation waits for `runtime_timeout` instead of giving up after 120 seconds, which is while the call is still running. Functions and services past the first page are found by `lithops runtime list` and `clean`. - [Knative] Calls longer than 5 minutes are no longer cut off by Knative, a load balancer ingress is reached at its own port and may be a hostname, `https` service URLs work, a failed call is reported instead of being invoked again for ever, and a service that never gets ready raises an error. ## [v3.8.0] diff --git a/docs/source/compute_config/openwhisk.md b/docs/source/compute_config/openwhisk.md index 96a3296aa..c111b0f17 100644 --- a/docs/source/compute_config/openwhisk.md +++ b/docs/source/compute_config/openwhisk.md @@ -46,6 +46,24 @@ docker run -d --name openwhisk -h openwhisk \ The image carries Docker 18.06 to launch action containers. `DOCKER_API_VERSION=1.40` lets that client talk to a current Docker daemon, which otherwise rejects it. Its Python runtimes are `python:3.10` and `python:3.11`, and an action is limited to 128–512 MB and 300 seconds. +On an Apple Silicon Mac that image is amd64 and its JVM crashes once invocations pile up. Run the same server jar on an arm64 Java container, with a Linux `docker` CLI. The Mac `docker` binary cannot run inside the container: + +```bash +docker create --name ow-jar openwhisk/standalone:nightly +docker cp ow-jar:/openwhisk-standalone.jar . +docker rm ow-jar +curl -sL https://download.docker.com/linux/static/stable/aarch64/docker-27.5.1.tgz | tar xz docker/docker + +docker run -d --name openwhisk -h openwhisk -p 3233:3233 \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v "$PWD/openwhisk-standalone.jar:/ow/openwhisk-standalone.jar:ro" \ + -v "$PWD/docker/docker:/usr/local/bin/docker:ro" \ + -w /ow eclipse-temurin:11-jre \ + bash -c 'exec java -Dwhisk.standalone.host.name=$(hostname) -Dwhisk.standalone.host.internal=$(hostname -i) -Dwhisk.standalone.host.external=localhost -jar openwhisk-standalone.jar --no-browser' +``` + +The guest key below is the same. Lithops picks `docker.io/lithopscloud/openwhisk-python-v310` through `v314` from the client Python version. Those images are built for amd64 and arm64. `lithops runtime build` builds for the machine it runs on; pass `--platform linux/amd64` when the cluster is amd64. + The playground is at . The guest namespace and its key are printed at startup (`docker logs openwhisk | grep 'wsk property'`) and are, by default: ```yaml diff --git a/lithops/executors.py b/lithops/executors.py index 4b25102c1..f1307ac8c 100644 --- a/lithops/executors.py +++ b/lithops/executors.py @@ -858,6 +858,9 @@ def wait( create_futures_list(fs_done, self), create_futures_list(fs_notdone, self), ) + # The same as any other failed wait: chunks still queued would + # otherwise be invoked after the futures were cancelled + self.invoker.discard_pending({f.job_key for f in futures}) self._cancel_wait(futures, do_clean, e) raise except KeyboardInterrupt as e: diff --git a/lithops/serverless/backends/aws_batch/aws_batch.py b/lithops/serverless/backends/aws_batch/aws_batch.py index 500aacb1a..ea5cc9c1a 100644 --- a/lithops/serverless/backends/aws_batch/aws_batch.py +++ b/lithops/serverless/backends/aws_batch/aws_batch.py @@ -177,18 +177,30 @@ def _create_compute_env(self): raise Exception('Compute env status must be VALID and state ENABLED') logger.debug(f'Using existing Compute Environment {self._compute_env_name}') - def _get_compute_env(self, ce_name=None): - res = self.batch_client.describe_compute_environments() + def _describe_all(self, method, item_key, **kwargs): + """Every item of a Batch describe call, following nextToken""" + items = [] + while True: + res = method(**kwargs) + if res['ResponseMetadata']['HTTPStatusCode'] != 200: + raise Exception(res) + items.extend(res[item_key]) + token = res.get('nextToken') + if not token: + return items + kwargs['nextToken'] = token - if res['ResponseMetadata']['HTTPStatusCode'] != 200: - raise Exception(res) + def _get_compute_env(self, ce_name=None): + compute_environments = self._describe_all( + self.batch_client.describe_compute_environments, 'computeEnvironments' + ) if ce_name is None: - compute_envs = [ce for ce in res['computeEnvironments'] if + compute_envs = [ce for ce in compute_environments if self.package in ce['computeEnvironmentName']] return compute_envs - compute_envs = [ce for ce in res['computeEnvironments'] if ce['computeEnvironmentName'] == ce_name] + compute_envs = [ce for ce in compute_environments if ce['computeEnvironmentName'] == ce_name] if len(compute_envs) == 0: return None if len(compute_envs) == 1: @@ -236,17 +248,14 @@ def _create_queue(self): logger.debug('Using existing Queue {}'.format(self._queue_name)) def _get_job_queue(self, jq_name=None): - res = self.batch_client.describe_job_queues() - - if res['ResponseMetadata']['HTTPStatusCode'] != 200: - raise Exception(res) + job_queues_all = self._describe_all(self.batch_client.describe_job_queues, 'jobQueues') if jq_name is None: - job_queues = [jq for jq in res['jobQueues'] + job_queues = [jq for jq in job_queues_all if self.package in jq['jobQueueName']] return job_queues - job_queues = [jq for jq in res['jobQueues'] if jq['jobQueueName'] == jq_name] + job_queues = [jq for jq in job_queues_all if jq['jobQueueName'] == jq_name] if len(job_queues) == 0: return None if len(job_queues) == 1: @@ -257,7 +266,6 @@ def _get_job_queue(self, jq_name=None): def _create_job_def(self, runtime_name, runtime_memory): job_def_name = self._format_jobdef_name(runtime_name, runtime_memory) - job_def = self._get_job_def(job_def_name) if self.env_type in {'EC2', 'SPOT'}: platform_capabilities = ['EC2'] @@ -274,58 +282,65 @@ def _create_job_def(self, runtime_name, runtime_memory): else: raise Exception(f'Unknown env type {self.env_type}') - if job_def is None: - logger.debug(f'Creating new Job Definition {job_def_name}') - image_name, _, _ = self._get_full_image_name(runtime_name) + # A new revision every time, including lithops runtime update. + # Reusing the existing definition left CPU, roles and the image as + # they were on the first deploy + previous = [ + jd for jd in self._active_job_defs() + if jd['jobDefinitionName'] == job_def_name + ] + logger.debug(f'Registering Job Definition {job_def_name}') + image_name, _, _ = self._get_full_image_name(runtime_name) + + container_properties = { + 'image': image_name, + 'executionRoleArn': self.aws_batch_config['execution_role'], + 'jobRoleArn': self.aws_batch_config['job_role'], + 'resourceRequirements': [ + { + 'type': 'VCPU', + 'value': str(self.aws_batch_config['runtime_cpu']) + }, + { + 'type': 'MEMORY', + 'value': str(runtime_memory) + } + ], + } - container_properties = { - 'image': image_name, - 'executionRoleArn': self.aws_batch_config['execution_role'], - 'jobRoleArn': self.aws_batch_config['job_role'], - 'resourceRequirements': [ - { - 'type': 'VCPU', - 'value': str(self.aws_batch_config['runtime_cpu']) - }, - { - 'type': 'MEMORY', - 'value': str(runtime_memory) - } - ], + if self.env_type in {'FARGATE', 'FARGATE_SPOT'}: + container_properties['networkConfiguration'] = { + 'assignPublicIp': 'ENABLED' if self.aws_batch_config['assign_public_ip'] else 'DISABLED' } - if self.env_type in {'FARGATE', 'FARGATE_SPOT'}: - container_properties['networkConfiguration'] = { - 'assignPublicIp': 'ENABLED' if self.aws_batch_config['assign_public_ip'] else 'DISABLED' - } + res = self.batch_client.register_job_definition( + jobDefinitionName=job_def_name, + type='container', + containerProperties=container_properties, + platformCapabilities=platform_capabilities + ) - res = self.batch_client.register_job_definition( - jobDefinitionName=job_def_name, - type='container', - containerProperties=container_properties, - platformCapabilities=platform_capabilities - ) + if res['ResponseMetadata']['HTTPStatusCode'] != 200: + raise Exception(res) + new_arn = res.get('jobDefinitionArn') + for old in previous: + if old['jobDefinitionArn'] != new_arn: + self._deregister_job_def(old['jobDefinitionArn']) + logger.debug('Job Definition {} successfully registered'.format(job_def_name)) + + def _active_job_defs(self): + return self._describe_all( + self.batch_client.describe_job_definitions, 'jobDefinitions', status='ACTIVE' + ) - if res['ResponseMetadata']['HTTPStatusCode'] != 200: - raise Exception(res) - logger.debug('Job Definition {} successfully created'.format(job_def_name)) - else: - if job_def['status'] != 'ACTIVE': - logger.error(job_def) - raise Exception('Job queue status must be VALID and state ENABLED') - logger.debug('Using existing Job Definition {}'.format(job_def_name)) + def _deregister_job_def(self, arn): + res = self.batch_client.deregister_job_definition(jobDefinition=arn) + if res['ResponseMetadata']['HTTPStatusCode'] != 200: + logger.error(res) + raise Exception(f'Could not deregister job definition {arn}') def _get_job_def(self, jd_name=None): - all_job_defs = [] - kwargs = {'status': 'ACTIVE'} - while True: - res = self.batch_client.describe_job_definitions(**kwargs) - if res['ResponseMetadata']['HTTPStatusCode'] != 200: - raise Exception(res) - all_job_defs.extend(res['jobDefinitions']) - if not res.get('nextToken'): - break - kwargs['nextToken'] = res['nextToken'] + all_job_defs = self._active_job_defs() if jd_name is None: job_defs = [jd for jd in all_job_defs @@ -333,13 +348,11 @@ def _get_job_def(self, jd_name=None): return job_defs job_defs = [jd for jd in all_job_defs if jd['jobDefinitionName'] == jd_name] - if len(job_defs) == 0: + if not job_defs: return None - if len(job_defs) == 1: - return job_defs.pop() - if len(job_defs) > 1: - logger.error(job_defs) - raise Exception('More than one job def with the same name') + # Several ACTIVE revisions share a name until the older ones are + # deregistered. The highest revision is the one just registered + return max(job_defs, key=lambda jd: jd.get('revision', 0)) def _generate_runtime_meta(self, runtime_name, runtime_memory): logger.info(f"Extracting metadata from: {runtime_name}") @@ -446,26 +459,24 @@ def deploy_runtime(self, runtime_name, memory, timeout=900): def delete_runtime(self, runtime_name, runtime_memory, version=__version__): jobdef_name = self._format_jobdef_name(runtime_name, runtime_memory, version) - job_def = self._get_job_def(jobdef_name) - if job_def is None: + named = [ + jd for jd in self._active_job_defs() + if jd['jobDefinitionName'] == jobdef_name + ] + if not named: logger.info(f'Job definition {jobdef_name} does not exist') return - logger.info('Deleting job definition with ARN {}'.format(job_def['jobDefinitionArn'])) - res = self.batch_client.deregister_job_definition(jobDefinition=job_def['jobDefinitionArn']) - if res['ResponseMetadata']['HTTPStatusCode'] != 200: - logger.error(res) - raise Exception('Could not deregister job definition {}'.format(job_def['jobDefinitionArn'])) + for job_def in named: + logger.info('Deleting job definition with ARN {}'.format(job_def['jobDefinitionArn'])) + self._deregister_job_def(job_def['jobDefinitionArn']) def clean(self, **kwargs): # Delete Job Definition job_defs = self._get_job_def() for job_def in job_defs: logger.info('Deregister job definition {}'.format(job_def['jobDefinitionArn'])) - res = self.batch_client.deregister_job_definition(jobDefinition=job_def['jobDefinitionArn']) - if res['ResponseMetadata']['HTTPStatusCode'] != 200: - logger.error(res) - raise Exception('Could not deregister job definition {}'.format(job_def['jobDefinitionArn'])) + self._deregister_job_def(job_def['jobDefinitionArn']) # Delete Job Queue job_queues = self._get_job_queue() diff --git a/lithops/serverless/backends/code_engine/code_engine.py b/lithops/serverless/backends/code_engine/code_engine.py index f6c6a1945..0349140b7 100644 --- a/lithops/serverless/backends/code_engine/code_engine.py +++ b/lithops/serverless/backends/code_engine/code_engine.py @@ -190,8 +190,7 @@ def _get_or_create_namespace(self, create=True, wait_for_active=True): self._create_code_engine_client() - response = self.ce_client.list_projects().get_result() - for project in response.get('projects', []): + for project in self._ce_list(self.ce_client.list_projects, 'projects'): if project['name'] == self.project_name: logger.debug(f"Found Code Engine project: {self.project_name}") self.project_id = project['id'] @@ -323,12 +322,25 @@ def _build_job_env_variables(self, action, payload_config_map_name=None): ) return env_variables + def _ce_list(self, method, item_key, *args): + """Every item of a Code Engine list, following the next.start token""" + items = [] + start = None + while True: + kwargs = {'start': start} if start else {} + page = method(*args, **kwargs).get_result() + items.extend(page.get(item_key, [])) + nxt = page.get('next') or {} + start = nxt.get('start') if isinstance(nxt, dict) else None + if not start: + return items + def _list_jobs(self): """ Lists all job definitions in the current project """ try: - return self.ce_client.list_jobs(self.project_id).get_result().get('jobs', []) + return self._ce_list(self.ce_client.list_jobs, 'jobs', self.project_id) except ApiException as e: logger.debug(f"List all jobs failed with {e.status_code} {e.message}") return [] @@ -409,14 +421,14 @@ def _delete_lithops_config_maps(self): Deletes leftover lithops config maps from the project """ try: - configmaps = self.ce_client.list_config_maps(self.project_id).get_result() + configmaps = self._ce_list(self.ce_client.list_config_maps, 'config_maps', self.project_id) except ApiException as e: logger.debug(f"Listing config maps failed with {e.status_code} {e.message}") return lithops_configmaps = [ configmap['name'] - for configmap in configmaps.get('config_maps', []) + for configmap in configmaps if configmap.get('name', '').startswith('lithops') ] if not lithops_configmaps: diff --git a/lithops/serverless/backends/gcp_cloudrun/cloudrun.py b/lithops/serverless/backends/gcp_cloudrun/cloudrun.py index 8f5510e5b..1fffeb532 100644 --- a/lithops/serverless/backends/gcp_cloudrun/cloudrun.py +++ b/lithops/serverless/backends/gcp_cloudrun/cloudrun.py @@ -316,11 +316,13 @@ def invoke(self, runtime_name, runtime_memory, payload, return_result=False): "Authorization": f"Bearer {id_token}", "Content-Type": "application/json" } + # The service runs the call inside this request and only then + # returns the activation id, so the client has to outlast runtime_timeout response = requests.post( service_url + route, data=json.dumps(payload, default=str), headers=headers, - timeout=120 + timeout=self.cr_config.get('runtime_timeout', 300) + 30 ) if response.status_code in (200, 202): @@ -501,17 +503,24 @@ def _delete_service(self, service_name): return raise + def _list_services(self): + """Every service in the project, following metadata.continue""" + services = [] + token = None + while True: + kwargs = {'parent': f'namespaces/{self.project_name}'} + if token: + kwargs['continue_'] = token + res = self._api_resource.namespaces().services().list(**kwargs).execute() + services.extend(res.get('items', [])) + token = (res.get('metadata') or {}).get('continue') + if not token: + return services + def clean(self, **kwargs): logger.debug('Going to delete all deployed runtimes') - res = self._api_resource.namespaces().services().list( - parent=f'namespaces/{self.project_name}', - ).execute() - - if 'items' not in res: - return - - for item in res['items']: + for item in self._list_services(): labels = item['spec']['template']['metadata']['labels'] if labels and 'type' in labels and labels['type'] == 'lithops-runtime': container = item['spec']['template']['spec']['containers'][0] @@ -523,15 +532,8 @@ def clean(self, **kwargs): def list_runtimes(self, runtime_name='all'): logger.debug('Listing runtimes') - res = self._api_resource.namespaces().services().list( - parent=f'namespaces/{self.project_name}', - ).execute() - - if 'items' not in res: - return [] - runtimes = [] - for item in res['items']: + for item in self._list_services(): labels = item['spec']['template']['metadata']['labels'] wk_name = item['metadata']['name'] if labels and 'type' in labels and labels['type'] == 'lithops-runtime': diff --git a/lithops/serverless/backends/gcp_functions/gcp_functions.py b/lithops/serverless/backends/gcp_functions/gcp_functions.py index 6da570d39..742b02f67 100644 --- a/lithops/serverless/backends/gcp_functions/gcp_functions.py +++ b/lithops/serverless/backends/gcp_functions/gcp_functions.py @@ -224,17 +224,13 @@ def _create_function(self, runtime_name, memory, timeout=60): logger.debug(f"Creating function {function_location}") - fn_list_response = self._api_resource.projects().locations().functions().list( - parent=self._default_location - ).execute(num_retries=self.num_retries) - if 'functions' in fn_list_response: - deployed_functions = [fn['name'] for fn in fn_list_response['functions']] - if function_location in deployed_functions: - logger.debug(f"Function {function_location} already exists - Deleting function") - self._api_resource.projects().locations().functions().delete( - name=function_location, - ).execute(num_retries=self.num_retries) - self._wait_function_deleted(function_location) + deployed_functions = [fn['name'] for fn in self._list_functions()] + if function_location in deployed_functions: + logger.debug(f"Function {function_location} already exists - Deleting function") + self._api_resource.projects().locations().functions().delete( + name=function_location, + ).execute(num_retries=self.num_retries) + self._wait_function_deleted(function_location) bin_location = self._get_runtime_bin_location(runtime_name) cloud_function = { @@ -420,14 +416,23 @@ def clean(self, **kwargs): self.delete_runtime( runtime_name, runtime_memory, version, function_name=wk_name) + def _list_functions(self): + """Every function in the region, following nextPageToken""" + functions = [] + page_token = None + while True: + response = self._api_resource.projects().locations().functions().list( + parent=self._default_location, pageToken=page_token + ).execute(num_retries=self.num_retries) + functions.extend(response.get('functions', [])) + page_token = response.get('nextPageToken') + if not page_token: + return functions + def list_runtimes(self, runtime_name='all'): logger.debug('Listing deployed runtimes') - response = self._api_resource.projects().locations().functions().list( - parent=self._default_location - ).execute(num_retries=self.num_retries) - runtimes = [] - for func in response.get('functions', []): + for func in self._list_functions(): if func['labels'] and 'type' in func['labels'] \ and func['labels']['type'] == 'lithops-runtime': fn_name = func['name'].rsplit('/', 1)[-1] @@ -489,11 +494,13 @@ def invoke(self, runtime_name, runtime_memory, payload={}, return_result=False): "Authorization": f"Bearer {api_token}", "Content-Type": "application/json" } + # The function runs inside this request and only then returns + # the activation id, so the client has to outlast runtime_timeout res = requests.post( function_url, data=json.dumps(payload, default=str), headers=headers, - timeout=120 + timeout=self.gcf_config.get('runtime_timeout', 300) + 30 ) if res.status_code in (200, 202): diff --git a/lithops/serverless/backends/openwhisk/openwhisk.py b/lithops/serverless/backends/openwhisk/openwhisk.py index 8b60580dc..894ff20b9 100644 --- a/lithops/serverless/backends/openwhisk/openwhisk.py +++ b/lithops/serverless/backends/openwhisk/openwhisk.py @@ -127,9 +127,9 @@ def build_runtime(self, docker_image_name, dockerfile, extra_args=[]): if dockerfile: assert os.path.isfile(dockerfile), f'Cannot locate "{dockerfile}"' - cmd = f'{docker_path} build --platform=linux/amd64 -t {docker_image_name} -f {dockerfile} . ' + cmd = f'{docker_path} build -t {docker_image_name} -f {dockerfile} . ' else: - cmd = f'{docker_path} build --platform=linux/amd64 -t {docker_image_name} . ' + cmd = f'{docker_path} build -t {docker_image_name} . ' cmd = cmd + ' '.join(extra_args) utils.run_command(cmd) diff --git a/lithops/serverless/backends/oracle_f/oracle_f.py b/lithops/serverless/backends/oracle_f/oracle_f.py index f9b68b6c7..faf6e9f0b 100644 --- a/lithops/serverless/backends/oracle_f/oracle_f.py +++ b/lithops/serverless/backends/oracle_f/oracle_f.py @@ -178,7 +178,7 @@ def clean(self, all=False): logger.debug(f'Application {self.app_name} does not exist') return - functions = self.cf_client.list_functions(self.app_id).data + functions = self._list_all(self.cf_client.list_functions, self.app_id) for function in functions: memory = function.memory_in_mbs @@ -404,8 +404,7 @@ def list_runtimes(self, runtime_name='all'): logger.debug(f'Application {self.app_name} does not exist') return runtimes - # Get the list of functions within the application - functions = self.cf_client.list_functions(self.app_id).data + functions = self._list_all(self.cf_client.list_functions, self.app_id) for function in functions: if function.display_name.startswith('lithops-worker'): @@ -417,6 +416,17 @@ def list_runtimes(self, runtime_name='all'): return runtimes + def _list_all(self, method, *args): + """Every item of an OCI list call, following opc-next-page""" + found = [] + page = None + while True: + response = method(*args, page=page) if page else method(*args) + found.extend(response.data or []) + page = getattr(response, 'next_page', None) + if not page: + return found + def get_runtime_info(self): """ Method that returns all the relevant information about the runtime set @@ -444,7 +454,7 @@ def _get_application_id(self, application_name): """ Returns the application id of a given application """ - applications = self.cf_client.list_applications(self.compartment_id).data + applications = self._list_all(self.cf_client.list_applications, self.compartment_id) for app in applications: if app.display_name == application_name: return app.id @@ -452,8 +462,7 @@ def _get_application_id(self, application_name): return None def _get_function_id(self, function_name): - # Get the list of functions within the application - functions = self.cf_client.list_functions(self.app_id).data + functions = self._list_all(self.cf_client.list_functions, self.app_id) # Search for the function and return its OCID for function in functions: diff --git a/lithops/standalone/backends/aws_ec2/aws_ec2.py b/lithops/standalone/backends/aws_ec2/aws_ec2.py index ded36cd4d..0a640eed3 100644 --- a/lithops/standalone/backends/aws_ec2/aws_ec2.py +++ b/lithops/standalone/backends/aws_ec2/aws_ec2.py @@ -812,6 +812,13 @@ def list_images(self): return sorted(result, key=lambda x: x[2], reverse=True) + def _iter_instances(self): + """Every instance, following NextToken. One page stops at 1,000""" + paginator = self.ec2_client.get_paginator('describe_instances') + for page in paginator.paginate(): + for reservation in page['Reservations']: + yield from reservation['Instances'] + def _delete_vm_instances(self, all=False): """ Deletes all worker VM instances @@ -823,15 +830,13 @@ def _delete_vm_instances(self, all=False): vms_prefixes = ('lithops-worker', 'lithops-master', 'building-image') if all else ('lithops-worker',) ins_to_delete = [] - response = self.ec2_client.describe_instances() - for res in response['Reservations']: - for ins in res['Instances']: - if ins['State']['Name'] != 'terminated' and 'Tags' in ins \ - and 'VpcId' in ins and self.ec2_data['vpc_id'] == ins['VpcId']: - for tag in ins['Tags']: - if tag['Key'] == 'Name' and tag['Value'].startswith(vms_prefixes): - ins_to_delete.append(ins['InstanceId']) - logger.debug(f"Going to delete VM instance {tag['Value']} ({ins['InstanceId']})") + for ins in self._iter_instances(): + if ins['State']['Name'] != 'terminated' and 'Tags' in ins \ + and 'VpcId' in ins and self.ec2_data['vpc_id'] == ins['VpcId']: + for tag in ins['Tags']: + if tag['Key'] == 'Name' and tag['Value'].startswith(vms_prefixes): + ins_to_delete.append(ins['InstanceId']) + logger.debug(f"Going to delete VM instance {tag['Value']} ({ins['InstanceId']})") if ins_to_delete: self.ec2_client.terminate_instances(InstanceIds=ins_to_delete) @@ -846,11 +851,9 @@ def _delete_vm_instances(self, all=False): while all and ins_to_delete: logger.debug('Waiting for VM instances to be terminated') status = set() - response = self.ec2_client.describe_instances() - for res in response['Reservations']: - for ins in res['Instances']: - if ins['InstanceId'] in ins_to_delete: - status.add(ins['State']['Name']) + for ins in self._iter_instances(): + if ins['InstanceId'] in ins_to_delete: + status.add(ins['State']['Name']) if len(status) == 1 and status.pop() == 'terminated': break else: diff --git a/lithops/tests/test_aws_backends.py b/lithops/tests/test_aws_backends.py index 169c1b60a..085f65fa4 100644 --- a/lithops/tests/test_aws_backends.py +++ b/lithops/tests/test_aws_backends.py @@ -112,6 +112,45 @@ def test_ec2_does_not_check_fargate_memory(self): kwargs = self._register_args(backend, 1536) assert kwargs['platformCapabilities'] == ['EC2'] + def test_an_existing_job_definition_is_replaced(self): + backend = _make_batch_backend(runtime_cpu=1) + name = backend._format_jobdef_name('rt:latest', 2048) + backend.batch_client.describe_job_definitions.return_value = {**OK, 'jobDefinitions': [{ + 'jobDefinitionName': name, 'jobDefinitionArn': 'arn:old', 'revision': 1, 'status': 'ACTIVE', + }]} + backend.batch_client.register_job_definition.return_value = {**OK, 'jobDefinitionArn': 'arn:new'} + backend.batch_client.deregister_job_definition.return_value = OK + backend._create_job_def('rt:latest', 2048) + resources = { + item['type']: item['value'] + for item in backend.batch_client.register_job_definition.call_args.kwargs[ + 'containerProperties' + ]['resourceRequirements'] + } + assert resources['VCPU'] == '1' + backend.batch_client.deregister_job_definition.assert_called_once_with(jobDefinition='arn:old') + + def test_job_queues_past_the_first_page(self): + backend = _make_batch_backend() + names = [f'{backend.package}_a', f'{backend.package}_b'] + backend.batch_client.describe_job_queues.side_effect = [ + {**OK, 'jobQueues': [{'jobQueueName': names[0]}], 'nextToken': 't'}, + {**OK, 'jobQueues': [{'jobQueueName': names[1]}]}, + ] + assert [queue['jobQueueName'] for queue in backend._get_job_queue()] == names + assert backend.batch_client.describe_job_queues.call_args_list[1].kwargs['nextToken'] == 't' + + def test_compute_environments_past_the_first_page(self): + backend = _make_batch_backend() + names = [f'{backend.package}_a', f'{backend.package}_b'] + backend.batch_client.describe_compute_environments.side_effect = [ + {**OK, 'computeEnvironments': [{'computeEnvironmentName': names[0]}], 'nextToken': 't'}, + {**OK, 'computeEnvironments': [{'computeEnvironmentName': names[1]}]}, + ] + found = [env['computeEnvironmentName'] for env in backend._get_compute_env()] + assert found == names + assert backend.batch_client.describe_compute_environments.call_args_list[1].kwargs['nextToken'] == 't' + class TestBatchJobDefinitionName: diff --git a/lithops/tests/test_azure_aliyun_oracle_backends.py b/lithops/tests/test_azure_aliyun_oracle_backends.py index f5802565d..e90558155 100644 --- a/lithops/tests/test_azure_aliyun_oracle_backends.py +++ b/lithops/tests/test_azure_aliyun_oracle_backends.py @@ -308,3 +308,37 @@ def invoke_function(function_id, invoke_function_body, fn_invoke_type): backend.invoke_functions.clear() assert backend.invoke('rt', 256, {}) == 'call-fn-256' assert invoked == [('https://fn-256', 'fn-256'), ('https://fn-1024', 'fn-1024'), ('https://fn-256', 'fn-256')] + + +def test_oracle_lists_functions_past_the_first_page(): + pytest.importorskip('oci') + from lithops.serverless.backends.oracle_f import oracle_f + + backend = oracle_f.OracleCloudFunctionsBackend.__new__(oracle_f.OracleCloudFunctionsBackend) + backend.app_id = 'app-id' + + def function(name): + return types.SimpleNamespace( + display_name=name, memory_in_mbs=256, image=f'img-{name}', + freeform_tags={'lithops_version': '3.8.1'}, + ) + + pages = { + None: types.SimpleNamespace( + data=[function('lithops-worker-a')], next_page='p2' + ), + 'p2': types.SimpleNamespace( + data=[function('lithops-worker-b')], next_page=None + ), + } + seen = [] + + def list_functions(app_id, page=None): + seen.append(page) + return pages[page] + + backend.cf_client = types.SimpleNamespace(list_functions=list_functions) + assert [row[3] for row in backend.list_runtimes()] == [ + 'lithops-worker-a', 'lithops-worker-b' + ] + assert seen == [None, 'p2'] diff --git a/lithops/tests/test_executors.py b/lithops/tests/test_executors.py index 2005155f5..c28658762 100644 --- a/lithops/tests/test_executors.py +++ b/lithops/tests/test_executors.py @@ -582,6 +582,18 @@ def test_wait_exception_drops_its_queued_calls_and_reraises(self, mock_wait): assert cleanup.call_args.kwargs['force'] is True assert isinstance(cleanup.call_args.kwargs['exception'], RuntimeError) + @patch('lithops.executors.wait', side_effect=TimeoutError('late')) + def test_a_timed_out_wait_drops_queued_invocations(self, mock_wait): + """ + Cancelling on timeout used to leave the queued chunks in the + invoker, which kept invoking them after the futures were dropped + """ + future = FakeFuture() + executor = _bare_executor() + with pytest.raises(TimeoutError): + executor.wait([future], show_progressbar=False) + executor.invoker.discard_pending.assert_called_once_with({future.job_key}) + @patch('lithops.executors.wait', side_effect=TimeoutError('late')) def test_a_failed_wait_only_cleans_the_jobs_it_waited_on(self, mock_wait): """ diff --git a/lithops/tests/test_ibm_openwhisk_backends.py b/lithops/tests/test_ibm_openwhisk_backends.py index 2c5da7919..f2f4b199a 100644 --- a/lithops/tests/test_ibm_openwhisk_backends.py +++ b/lithops/tests/test_ibm_openwhisk_backends.py @@ -110,6 +110,19 @@ def test_openwhisk_clean_deletes_each_action_once(): backend.cf_client.delete_package.assert_called_once_with(backend.package) +def test_openwhisk_build_runtime_uses_the_host_architecture(monkeypatch): + from lithops.serverless.backends.openwhisk import openwhisk + backend = _make_ow_backend() + backend.ow_config = {} + commands = [] + monkeypatch.setattr(openwhisk.utils, 'get_docker_path', lambda: 'docker') + monkeypatch.setattr(openwhisk.utils, 'is_podman', lambda path: False) + monkeypatch.setattr(openwhisk.utils, 'run_command', commands.append) + backend.build_runtime('user/rt:1', None, ['--platform', 'linux/amd64']) + assert commands[0] == 'docker build -t user/rt:1 . --platform linux/amd64' + assert '--platform=linux/amd64' not in commands[0] + + def test_openwhisk_runtime_meta_waits_once(): backend = _make_ow_backend() backend.cf_client.invoke_with_result.return_value = {'preinstalls': ['requests']} @@ -142,6 +155,15 @@ def ce(): return types.SimpleNamespace(backend=backend, module=code_engine, ApiException=ApiException) +def test_code_engine_lists_jobs_past_the_first_page(ce): + ce.backend.ce_client.list_jobs.side_effect = [ + MagicMock(**{'get_result.return_value': {'jobs': [{'name': 'a'}], 'next': {'start': 's'}}}), + MagicMock(**{'get_result.return_value': {'jobs': [{'name': 'b'}]}}), + ] + assert [job['name'] for job in ce.backend._list_jobs()] == ['a', 'b'] + assert ce.backend.ce_client.list_jobs.call_args_list[1].kwargs['start'] == 's' + + def test_code_engine_clear_removes_all_jobs(ce, monkeypatch): monkeypatch.setattr(ce.backend, '_get_or_create_namespace', MagicMock(return_value='ns')) ce.backend.jobs = ['A1-00', 'A1-01', 'A1-02', 'A1-03'] diff --git a/lithops/tests/test_serverless.py b/lithops/tests/test_serverless.py index d6945b2ad..c11d1a66a 100644 --- a/lithops/tests/test_serverless.py +++ b/lithops/tests/test_serverless.py @@ -439,6 +439,54 @@ def test_the_metadata_error_survives_a_failed_cleanup(self, tmp_path, monkeypatc with pytest.raises(Exception, match='metadata down'): backend.deploy_runtime('rt', 256, 60) + def test_https_invoke_waits_as_long_as_the_runtime(self, monkeypatch): + backend = self._backend() + backend.gcf_config = {'runtime_timeout': 300} + backend._format_function_name = lambda runtime_name, memory: 'fn' + backend._get_token = lambda function_name: ('https://fn', 'tok') + seen = {} + + def post(url, data, headers, timeout): + seen['timeout'] = timeout + return SimpleNamespace(status_code=202, json=lambda: {'activationId': 'act'}) + + monkeypatch.setattr( + 'lithops.serverless.backends.gcp_functions.gcp_functions.requests.post', post + ) + assert backend.invoke('rt', 256, {}) == 'act' + assert seen['timeout'] == 330 + + def test_functions_past_the_first_page_are_listed(self): + pytest.importorskip('googleapiclient') + from lithops.serverless.backends.gcp_functions.gcp_functions import GCPFunctionsBackend + backend = GCPFunctionsBackend.__new__(GCPFunctionsBackend) + backend.num_retries = 0 + backend.project_name = 'p' + backend.region = 'l' + backend._api_resource = MagicMock() + list_call = ( + backend._api_resource.projects.return_value.locations.return_value + .functions.return_value.list + ) + + def fn(name): + return { + 'name': f'projects/p/locations/l/functions/{name}', + 'labels': { + 'type': 'lithops-runtime', + 'lithops_version': '3-8-1-dev0', + 'runtime_name': 'rt', + }, + 'serviceConfig': {'availableMemory': '256M'}, + } + + list_call.return_value.execute.side_effect = [ + {'functions': [fn('a')], 'nextPageToken': 'tok'}, + {'functions': [fn('b')]}, + ] + assert [row[3] for row in backend.list_runtimes()] == ['a', 'b'] + assert list_call.call_args_list[1].kwargs['pageToken'] == 'tok' + class TestCloudRunDeployAndDelete: @@ -502,3 +550,46 @@ def test_a_failed_delete_is_raised(self): with pytest.raises(Exception): backend._delete_service('svc') services.get.assert_not_called() + + def test_invoke_waits_as_long_as_the_runtime(self, monkeypatch): + backend = self._backend() + backend.cr_config = {'runtime_timeout': 300} + backend._format_image_name = lambda runtime_name: 'img' + backend._format_service_name = lambda img_name, memory: 'svc' + backend._get_url_and_token = lambda service_name: ('https://svc', 'tok') + seen = {} + + def post(url, data, headers, timeout): + seen['timeout'] = timeout + return SimpleNamespace(status_code=202, json=lambda: {'activationId': 'act'}) + + monkeypatch.setattr( + 'lithops.serverless.backends.gcp_cloudrun.cloudrun.requests.post', post + ) + assert backend.invoke('rt', 256, {}) == 'act' + assert seen['timeout'] == 330 + + def test_services_past_the_first_page_are_listed(self): + backend = self._backend() + services = backend._api_resource.namespaces.return_value.services.return_value + + def item(name): + return { + 'metadata': {'name': name}, + 'spec': {'template': { + 'metadata': {'labels': { + 'type': 'lithops-runtime', 'lithops-version': '3-8-1-dev0', + }}, + 'spec': {'containers': [{ + 'image': f'img-{name}', + 'resources': {'limits': {'memory': '256Mi'}}, + }]}, + }}, + } + + services.list.return_value.execute.side_effect = [ + {'items': [item('a')], 'metadata': {'continue': 'tok'}}, + {'items': [item('b')]}, + ] + assert [row[3] for row in backend.list_runtimes()] == ['a', 'b'] + assert services.list.call_args_list[1].kwargs['continue_'] == 'tok' diff --git a/lithops/tests/test_standalone.py b/lithops/tests/test_standalone.py index 08fc8a10c..768cc1c81 100644 --- a/lithops/tests/test_standalone.py +++ b/lithops/tests/test_standalone.py @@ -748,6 +748,35 @@ def test_aws_spot_price_without_history_raises(self): with pytest.raises(ValueError, match='No spot price history for worker instance type t2.micro'): backend._request_spot_price() + def test_delete_vms_reads_every_page_of_instances(self): + from lithops.standalone.backends.aws_ec2.aws_ec2 import AWSEC2Backend + backend = AWSEC2Backend.__new__(AWSEC2Backend) + backend.vpc_name = 'lithops-vpc' + backend.ec2_data = { + 'vpc_id': 'vpc-1', 'vpc_data_type': 'provided', 'master_name': 'master', + } + backend.cache_dir = '/tmp' + backend.ec2_client = MagicMock() + + def instance(instance_id, name): + return { + 'InstanceId': instance_id, + 'State': {'Name': 'running'}, + 'VpcId': 'vpc-1', + 'Tags': [{'Key': 'Name', 'Value': name}], + } + + backend.ec2_client.get_paginator.return_value.paginate.return_value = [ + {'Reservations': [{'Instances': [instance('i-1', 'lithops-worker-a')]}]}, + {'Reservations': [{'Instances': [ + instance('i-2', 'lithops-worker-b'), instance('i-3', 'other'), + ]}]}, + ] + backend._delete_vm_instances() + backend.ec2_client.terminate_instances.assert_called_once_with( + InstanceIds=['i-1', 'i-2'] + ) + def test_ibm_deletes_a_local_ssh_key_that_could_not_be_registered(self, tmp_path): backend, load_data, _, _, _, deletes = _ibm_backend_for_init() from ibm_cloud_sdk_core import ApiException diff --git a/runtime/openwhisk/README.md b/runtime/openwhisk/README.md index 498a4e3b3..8c319694a 100644 --- a/runtime/openwhisk/README.md +++ b/runtime/openwhisk/README.md @@ -53,6 +53,8 @@ fexec = lithops.FunctionExecutor(runtime_memory=512) $ lithops runtime build -b openwhisk docker_username/runtimename:tag --build-arg PYTHON_VERSION=3.13 + The image is built for the architecture of the machine running the command. Pass `--platform linux/amd64` when the OpenWhisk cluster is amd64 and this machine is not. + Note that Docker Hub image names look like *"docker_username/runtimename:tag"* and must be all lower case, for example: $ lithops runtime build -b openwhisk jsampe/lithops-custom-runtime-v312:0.1 From a1bd1ae4e697e5a9bed82d147c2b093598fc32c5 Mon Sep 17 00:00:00 2001 From: JosepSampe Date: Sun, 25 Oct 2026 15:00:57 +0100 Subject: [PATCH 06/13] Fix backends --- CHANGELOG.md | 4 +- .../backends/aws_lambda/aws_lambda.py | 12 +- .../gcp_compute_engine/gcp_compute_engine.py | 17 ++- .../standalone/backends/ibm_vpc/ibm_vpc.py | 89 ++++++++++----- lithops/tests/test_aws_backends.py | 14 +++ lithops/tests/test_standalone.py | 107 ++++++++++++++++++ 6 files changed, 202 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b2151c695..46248c9bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,8 +24,8 @@ - [Core] An executor that fails to be created no longer logs a misleading `job_monitor` error at exit. - [Core] The FaaS invoker no longer runs up to two workers beyond `max_workers`. A `wait()` that times out drops the invocations still queued, the same as any other failed wait. `obj_chunk_number` splits an object into exactly that many chunks, or fewer non-empty ones when it is smaller. URL inputs are sized following redirects, an HTTP error raises, and only `Accept-Ranges: bytes` enables partitioning. A local input path that does not exist raises `FileNotFoundError`, and local inputs with no files raise like an empty bucket. A mode section with no body in the config file no longer crashes config loading. - [CLI] `lithops runtime update` stores `runtime_timeout` in the runtime metadata, which every later job needed. -- [Standalone] Cleaning AWS EC2 workers reads every page of instances, so workers past the first 1,000 are stopped too. `get_result()` on one job no longer cancels the other jobs of the executor, and create mode stops its workers only once none of its jobs is left. Worker VMs whose setup fails, and the workers of a job whose master cannot be set up, are stopped instead of left running. Requests to the master from inside a worker send their body, and an error from the master raises. They wait up to 60 seconds for an answer instead of 1, and as long as the master needs to pull the runtime for its metadata; a POST no longer waits for ever. The `vm` backend runs jobs: it reads `ssh_username` and provides what the master and its budget keeper need. Azure VMs are deallocated when stopped, so they are no longer billed, the home directory follows `ssh_username`, and `delete_on_dismantle` defaults to `False` as documented. -- [AWS Lambda] Invocations retry connection errors a few times, re-signing each attempt, instead of retrying any error for ever. `lithops runtime update` updates the code, configuration and tags of a deployed function, and the default runtime gets the `user_tags` too. `env_vars` and `user_tags` accept a `{NAME: VALUE}` mapping or a list of `{name, value}`. Layers with more than 50 versions are deleted. Runtimes and layers past the first page of the listing are found by `lithops runtime list` and `clean`, and deleting a runtime whose function is already gone works. +- [Standalone] Cleaning AWS EC2 workers reads every page of instances, so workers past the first 1,000 are stopped too. Cleaning IBM VPC workers, choosing its image and reusing a free floating IP read every page, which stops at 50, as do the VPC, subnet, gateway and SSH key lookups. Cleaning GCP Compute Engine workers reads every page of instances, which stops at 500. `get_result()` on one job no longer cancels the other jobs of the executor, and create mode stops its workers only once none of its jobs is left. Worker VMs whose setup fails, and the workers of a job whose master cannot be set up, are stopped instead of left running. Requests to the master from inside a worker send their body, and an error from the master raises. They wait up to 60 seconds for an answer instead of 1, and as long as the master needs to pull the runtime for its metadata; a POST no longer waits for ever. The `vm` backend runs jobs: it reads `ssh_username` and provides what the master and its budget keeper need. Azure VMs are deallocated when stopped, so they are no longer billed, the home directory follows `ssh_username`, and `delete_on_dismantle` defaults to `False` as documented. +- [AWS Lambda] Invocations retry connection errors a few times, re-signing each attempt, instead of retrying any error for ever. `lithops runtime update` updates the code, configuration and tags of a deployed function, and the default runtime gets the `user_tags` too. `env_vars` and `user_tags` accept a `{NAME: VALUE}` mapping or a list of `{name, value}`. Layers with more than 50 versions are deleted. Runtimes and layers past the first page of the listing are found by `lithops runtime list` and `clean`, and deleting a runtime whose function is already gone works. A container runtime is looked up by its image tag, so it is found in a repository with more than 100 images. - [AWS Batch] Maps over more than 20 calls no longer fail on a fractional `max_workers`. Job definitions use the requested `runtime_memory`, checked against the Fargate combinations, and their names carry the requested version, so `lithops runtime update` and `delete` find them, also past the first page of job definitions. Deleting a runtime whose job definition is already gone works. Job queues and compute environments past the first page are found, and `lithops runtime update` registers a new job definition revision instead of keeping the CPU, roles and image from the first deploy. - [OpenWhisk] An invoke rejected with 401 or 404 raises instead of being reported as an activation id, which made `wait()` hang. A failure to read runtime metadata raises that error instead of a `TypeError`. Python 3.13 and 3.14 clients get a default runtime, built from the runtime Dockerfile with `--build-arg PYTHON_VERSION`. The client speaks to current OpenWhisk (tested with `openwhisk/standalone:nightly`): an `http://` endpoint is used as configured, actions and packages past the first page are listed, a package that already exists is reused, deleting a missing action works, and waiting for runtime metadata follows the activation instead of invoking again for ever. A non-blocking invocation opens its own connection and closes it, so the invoker threads do not queue on the requests connection pool. `lithops runtime build` builds for the machine it runs on; `--platform linux/amd64` selects amd64. - [Code Engine] Projects, job definitions and config maps past the first page are listed. `clear()` deletes every job run and config map of the executor, a leftover config map is replaced instead of failing every deploy, and waiting for a job run or a job definition deletion times out. diff --git a/lithops/serverless/backends/aws_lambda/aws_lambda.py b/lithops/serverless/backends/aws_lambda/aws_lambda.py index e8e9b0252..9ddc88c85 100644 --- a/lithops/serverless/backends/aws_lambda/aws_lambda.py +++ b/lithops/serverless/backends/aws_lambda/aws_lambda.py @@ -514,13 +514,17 @@ def _deploy_container_runtime(self, runtime_name, memory, timeout): try: repo_name = self._format_repo_name(image) - response = self.ecr_client.describe_images(repositoryName=repo_name) - images = response['imageDetails'] + # Ask for the tag. A plain describe stops at 100 images, so the + # tag can be missing from the first page of a busy repository + response = self.ecr_client.describe_images( + repositoryName=repo_name, + imageIds=[{'imageTag': tag}], + ) + images = response.get('imageDetails') or [] if not images: raise Exception(f'Runtime {runtime_name} is not present in ECR.' 'Consider running "lithops runtime build -b aws_lambda ..."') - image = list(filter(lambda image: 'imageTags' in image and tag in image['imageTags'], images)).pop() - image_digest = image['imageDigest'] + image_digest = images[0]['imageDigest'] except botocore.exceptions.ClientError: raise Exception(f'Runtime "{runtime_name}" is not deployed to ECR') diff --git a/lithops/standalone/backends/gcp_compute_engine/gcp_compute_engine.py b/lithops/standalone/backends/gcp_compute_engine/gcp_compute_engine.py index a6dd96467..59c371f41 100644 --- a/lithops/standalone/backends/gcp_compute_engine/gcp_compute_engine.py +++ b/lithops/standalone/backends/gcp_compute_engine/gcp_compute_engine.py @@ -772,6 +772,18 @@ def _format_image_timestamp(timestamp): return timestamp return created_at.strftime('%Y-%m-%d %H:%M:%S') + def _iter_zone_instances(self): + """Every instance in the zone. One page stops at 500""" + request = self.compute_client.instances().list( + project=self.project_name, zone=self.zone + ) + while request is not None: + response = request.execute() + yield from response.get('items') or [] + request = self.compute_client.instances().list_next( + previous_request=request, previous_response=response + ) + def _iter_project_images(self, project): request = self.compute_client.images().list(project=project) while request is not None: @@ -877,11 +889,8 @@ def _delete_vm_instances(self, all=False): ) def get_instance_names(): - instances = self.compute_client.instances().list( - project=self.project_name, zone=self.zone - ).execute().get('items', []) or [] return [ - ins['name'] for ins in instances + ins['name'] for ins in self._iter_zone_instances() if ins.get('name', '').startswith(prefixes) ] diff --git a/lithops/standalone/backends/ibm_vpc/ibm_vpc.py b/lithops/standalone/backends/ibm_vpc/ibm_vpc.py index 58070e3b1..16d7676bb 100644 --- a/lithops/standalone/backends/ibm_vpc/ibm_vpc.py +++ b/lithops/standalone/backends/ibm_vpc/ibm_vpc.py @@ -23,6 +23,7 @@ import functools import inspect from datetime import datetime +from urllib.parse import parse_qs, urlparse from ibm_vpc import VpcV1 from ibm_cloud_sdk_core.authenticators import IAMAuthenticator from ibm_cloud_sdk_core import ApiException @@ -237,8 +238,7 @@ def _create_vpc(self): assert re.match("^[a-z0-9-:-]*$", self.vpc_name), \ f'VPC name "{self.vpc_name}" not valid' - vpcs_info = self.vpc_cli.list_vpcs().get_result() - for vpc in vpcs_info['vpcs']: + for vpc in _list_all(self.vpc_cli.list_vpcs, 'vpcs'): if vpc['name'] == self.vpc_name: vpc_info = vpc @@ -311,7 +311,7 @@ def _create_ssh_key(self): key_info = None def _get_ssh_key(): - for key in self.vpc_cli.list_keys().result["keys"]: + for key in _list_all(self.vpc_cli.list_keys, 'keys'): if key["name"] == keyname: return key @@ -375,8 +375,10 @@ def _create_subnet(self): subnet_name = f'lithops-subnet-{self.vpc_key}' subnet_data = None - subnets_info = self.vpc_cli.list_subnets(resource_group_id=self.config['resource_group_id']).get_result() - for sn in subnets_info['subnets']: + for sn in _list_all( + self.vpc_cli.list_subnets, 'subnets', + resource_group_id=self.config['resource_group_id'], + ): if sn['name'] == subnet_name: subnet_data = sn @@ -415,8 +417,7 @@ def _create_gateway(self): gateway_name = f'lithops-gateway-{self.vpc_key}' gateway_data = None - gateways_info = self.vpc_cli.list_public_gateways().get_result() - for gw in gateways_info['public_gateways']: + for gw in _list_all(self.vpc_cli.list_public_gateways, 'public_gateways'): if gw['vpc']['id'] == self.config['vpc_id']: gateway_data = gw @@ -445,8 +446,7 @@ def _get_or_create_floating_ip(self): """ fip_data = None - floating_ips_info = self.vpc_cli.list_floating_ips().get_result() - for fip in floating_ips_info['floating_ips']: + for fip in _list_all(self.vpc_cli.list_floating_ips, 'floating_ips'): if fip['name'].startswith("lithops-recyclable") and 'target' not in fip \ and fip['zone']['name'] == self.config['zone_name']: fip_data = fip @@ -500,9 +500,11 @@ def _request_image_id(self): self.config['image_id'] = self.vpc_data['image_id'] return - images_def = self.vpc_cli.list_images().result['images'] - images_user = self.vpc_cli.list_images(resource_group_id=self.config['resource_group_id']).result['images'] - images_def.extend(images_user) + images_def = _list_all(self.vpc_cli.list_images, 'images') + images_def.extend(_list_all( + self.vpc_cli.list_images, 'images', + resource_group_id=self.config['resource_group_id'], + )) if 'image_id' in self.vpc_data: for image in images_def: @@ -626,10 +628,10 @@ def build_image(self, image_name, script_file, overwrite, include, extra_args=[] """ image_name = image_name or DEFAULT_LITHOPS_IMAGE_NAME - images = self.vpc_cli.list_images( - name=image_name, - resource_group_id=self.config['resource_group_id'] - ).result['images'] + images = _list_all( + self.vpc_cli.list_images, 'images', + name=image_name, resource_group_id=self.config['resource_group_id'], + ) if len(images) > 0: image_id = images[0]['id'] @@ -707,9 +709,10 @@ def build_image(self, image_name, script_file, overwrite, include, extra_args=[] logger.debug("Be patient, VM imaging can take up to 6 minutes") while True: - images = self.vpc_cli.list_images( - name=image_name, resource_group_id=self.config['resource_group_id'] - ).result['images'] + images = _list_all( + self.vpc_cli.list_images, 'images', + name=image_name, resource_group_id=self.config['resource_group_id'], + ) if len(images) > 0: logger.debug(f"VM Image is being created. Current status: {images[0]['status']}") if images[0]['status'] == 'available': @@ -728,10 +731,10 @@ def delete_image(self, image_name): Deletes a VM Image """ def list_images(): - return self.vpc_cli.list_images( - name=image_name, - resource_group_id=self.config['resource_group_id'] - ).result['images'] + return _list_all( + self.vpc_cli.list_images, 'images', + name=image_name, resource_group_id=self.config['resource_group_id'], + ) images = list_images() @@ -794,9 +797,11 @@ def list_images(self): List Ubuntu LTS images (22.04–24.04) and Lithops custom images. Returns tuples of (name, image_id, creation_date). """ - images_def = self.vpc_cli.list_images().result['images'] - images_user = self.vpc_cli.list_images(resource_group_id=self.config['resource_group_id']).result['images'] - images_def.extend(images_user) + images_def = _list_all(self.vpc_cli.list_images, 'images') + images_def.extend(_list_all( + self.vpc_cli.list_images, 'images', + resource_group_id=self.config['resource_group_id'], + )) result = set() @@ -835,8 +840,7 @@ def delete_instance(instance_info): def get_instances(): instances = set() - instances_info = self.vpc_cli.list_instances().get_result() - for ins in instances_info['instances']: + for ins in _list_all(self.vpc_cli.list_instances, 'instances'): if ins['name'].startswith(vms_prefixes) \ and ins['vpc']['id'] == self.vpc_data['vpc_id']: instances.add((ins['name'], ins['id'])) @@ -1242,9 +1246,9 @@ def get_instance_data(self): """ Returns the instance information """ - instances_data = self.vpc_cli.list_instances(name=self.name).get_result() - if len(instances_data['instances']) > 0: - self.instance_data = instances_data['instances'][0] + instances_data = _list_all(self.vpc_cli.list_instances, 'instances', name=self.name) + if instances_data: + self.instance_data = instances_data[0] self.instance_id = self.instance_data['id'] return self.instance_data @@ -1424,6 +1428,29 @@ def validate_capabilities(self): ) +def _page_start(next_page): + """ + The start token of the next VPC page. next.href carries it as a query + parameter, and a page stops at 50 items + """ + href = (next_page or {}).get('href') + if not href: + return None + return (parse_qs(urlparse(href).query).get('start') or [None])[0] + + +def _list_all(method, item_key, **kwargs): + """Every item of a VPC list, following next.href""" + found = [] + while True: + result = method(**kwargs).get_result() + found.extend(result.get(item_key) or []) + start = _page_start(result.get('next')) + if not start: + return found + kwargs['start'] = start + + def _status_code(err): """ The HTTP status of an ApiException. status_code replaced the deprecated diff --git a/lithops/tests/test_aws_backends.py b/lithops/tests/test_aws_backends.py index 085f65fa4..cd4d050a4 100644 --- a/lithops/tests/test_aws_backends.py +++ b/lithops/tests/test_aws_backends.py @@ -337,6 +337,20 @@ def test_default_runtime_updates_existing_function(self, monkeypatch): assert conf['Layers'] == ['arn:layer'] assert conf['Environment'] == {'Variables': {'A': 'b'}} + def test_container_runtime_is_looked_up_by_its_tag(self): + """A plain describe_images stops at 100, so the tag can be on a later page""" + backend = _make_lambda_backend() + backend.ecr_client.describe_images.return_value = { + 'imageDetails': [{'imageDigest': 'sha256:new'}] + } + backend._deploy_container_runtime('myrt:v2', 1024, 300) + backend.ecr_client.describe_images.assert_called_once_with( + repositoryName=backend._format_repo_name('myrt'), + imageIds=[{'imageTag': 'v2'}], + ) + code = backend.lambda_client.create_function.call_args.kwargs + assert code['Code']['ImageUri'].endswith('@sha256:new') + def test_container_runtime_updates_existing_function(self): backend = _make_lambda_backend() backend.ecr_client.describe_images.return_value = { diff --git a/lithops/tests/test_standalone.py b/lithops/tests/test_standalone.py index 768cc1c81..20e01c915 100644 --- a/lithops/tests/test_standalone.py +++ b/lithops/tests/test_standalone.py @@ -777,6 +777,113 @@ def instance(instance_id, name): InstanceIds=['i-1', 'i-2'] ) + def test_gcp_clean_reads_every_page_of_instances(self): + from lithops.standalone.backends.gcp_compute_engine.gcp_compute_engine import GCPComputeEngineBackend + backend = GCPComputeEngineBackend.__new__(GCPComputeEngineBackend) + backend.network_name = 'lithops-net' + backend.project_name = 'project' + backend.zone = 'us-east1-b' + backend.compute_client = MagicMock() + backend._wait_operation = MagicMock() + + first = MagicMock() + first.execute.return_value = { + 'items': [{'name': 'lithops-worker-a'}], 'nextPageToken': 'tok', + } + second = MagicMock() + second.execute.return_value = {'items': [ + {'name': 'lithops-worker-b'}, {'name': 'other'}, + ]} + empty = MagicMock() + empty.execute.return_value = {'items': []} + instances = backend.compute_client.instances.return_value + instances.list.side_effect = [first, empty] + instances.list_next.side_effect = [second, None, None] + instances.delete.return_value.execute.return_value = {'name': 'op'} + + backend._delete_vm_instances() + + deleted = [ + call.kwargs['instance'] + for call in instances.delete.call_args_list + ] + assert deleted == ['lithops-worker-a', 'lithops-worker-b'] + + @staticmethod + def _vpc_page(items, key, start=None): + result = {key: items} + if start: + result['next'] = { + 'href': f'https://example/v1/{key}?limit=50&start={start}' + } + response = MagicMock() + response.get_result.return_value = result + return response + + def test_ibm_clean_reads_every_page_of_instances(self): + from lithops.standalone.backends.ibm_vpc.ibm_vpc import IBMVPCBackend + backend = IBMVPCBackend.__new__(IBMVPCBackend) + backend.vpc_name = 'lithops-vpc' + backend.vpc_data = { + 'vpc_id': 'vpc-1', 'vpc_data_type': 'provided', 'master_name': 'master', + } + backend.cache_dir = '/tmp' + backend.vpc_cli = MagicMock() + backend.vpc_cli.list_instances.side_effect = [ + self._vpc_page( + [{'name': 'lithops-worker-a', 'id': 'i-1', 'vpc': {'id': 'vpc-1'}}], + 'instances', start='tok', + ), + self._vpc_page([ + {'name': 'lithops-worker-b', 'id': 'i-2', 'vpc': {'id': 'vpc-1'}}, + {'name': 'other', 'id': 'i-3', 'vpc': {'id': 'vpc-1'}}, + ], 'instances'), + self._vpc_page([], 'instances'), + ] + + backend._delete_vm_instances() + + assert backend.vpc_cli.list_instances.call_args_list[1].kwargs['start'] == 'tok' + deleted = {call.args[0] for call in backend.vpc_cli.delete_instance.call_args_list} + assert deleted == {'i-1', 'i-2'} + + def test_ibm_image_lookup_reads_the_page_after_the_first(self): + from lithops.standalone.backends.ibm_vpc.ibm_vpc import IBMVPCBackend + backend = IBMVPCBackend.__new__(IBMVPCBackend) + backend.config = {'resource_group_id': 'rg'} + backend.vpc_data = {} + backend.vpc_cli = MagicMock() + backend.vpc_cli.list_images.side_effect = [ + self._vpc_page([{'id': 'img-win', 'name': 'ibm-windows-1'}], 'images', start='tok'), + self._vpc_page( + [{'id': 'img-ubuntu', 'name': 'ibm-ubuntu-24-04-amd64-1'}], 'images' + ), + self._vpc_page([], 'images'), + ] + + backend._request_image_id() + + assert backend.config['image_id'] == 'img-ubuntu' + assert backend.vpc_cli.list_images.call_args_list[1].kwargs['start'] == 'tok' + + def test_ibm_reuses_a_floating_ip_past_the_first_page(self): + from lithops.standalone.backends.ibm_vpc.ibm_vpc import IBMVPCBackend + backend = IBMVPCBackend.__new__(IBMVPCBackend) + backend.config = {'zone_name': 'eu-de-1', 'resource_group_id': 'rg'} + backend.vpc_cli = MagicMock() + free = { + 'name': 'lithops-recyclable-abcd', 'id': 'fip-1', 'address': '1.2.3.4', + 'zone': {'name': 'eu-de-1'}, + } + backend.vpc_cli.list_floating_ips.side_effect = [ + self._vpc_page([], 'floating_ips', start='tok'), + self._vpc_page([free], 'floating_ips'), + ] + + assert backend._get_or_create_floating_ip() == ('1.2.3.4', 'fip-1') + backend.vpc_cli.create_floating_ip.assert_not_called() + assert backend.vpc_cli.list_floating_ips.call_args_list[1].kwargs['start'] == 'tok' + def test_ibm_deletes_a_local_ssh_key_that_could_not_be_registered(self, tmp_path): backend, load_data, _, _, _, deletes = _ibm_backend_for_init() from ibm_cloud_sdk_core import ApiException From eb5e4ab025b8d1d7ac30ee5ed03e96e5468aee7a Mon Sep 17 00:00:00 2001 From: JosepSampe Date: Sun, 25 Oct 2026 15:21:05 +0100 Subject: [PATCH 07/13] Fix backneds --- CHANGELOG.md | 5 +-- .../backends/aliyun_fc/aliyun_fc.py | 21 +++++++++++-- .../azure_functions/azure_functions.py | 12 +++++-- lithops/serverless/backends/k8s/k8s.py | 19 ++++++------ .../test_azure_aliyun_oracle_backends.py | 31 +++++++++++++++++-- lithops/tests/test_k8s_backend.py | 10 ++++++ 6 files changed, 78 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46248c9bd..a6ebf9180 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,9 +30,10 @@ - [OpenWhisk] An invoke rejected with 401 or 404 raises instead of being reported as an activation id, which made `wait()` hang. A failure to read runtime metadata raises that error instead of a `TypeError`. Python 3.13 and 3.14 clients get a default runtime, built from the runtime Dockerfile with `--build-arg PYTHON_VERSION`. The client speaks to current OpenWhisk (tested with `openwhisk/standalone:nightly`): an `http://` endpoint is used as configured, actions and packages past the first page are listed, a package that already exists is reused, deleting a missing action works, and waiting for runtime metadata follows the activation instead of invoking again for ever. A non-blocking invocation opens its own connection and closes it, so the invoker threads do not queue on the requests connection pool. `lithops runtime build` builds for the machine it runs on; `--platform linux/amd64` selects amd64. - [Code Engine] Projects, job definitions and config maps past the first page are listed. `clear()` deletes every job run and config map of the executor, a leftover config map is replaced instead of failing every deploy, and waiting for a job run or a job definition deletion times out. - [Singularity] Runtime metadata that never arrives raises the intended timeout error. -- [Azure Functions] An invoke that returns an HTTP error other than 429 raises. Waiting for runtime metadata with the pub/sub trigger times out after `runtime_timeout`. `runtime_memory` is honoured after the first deploy: each Flex instance memory size gets its own function app, so existing runtimes are deployed again once per size. `lithops runtime update` and `delete` find a runtime by its name. +- [Kubernetes] Worker memory is compared in megabytes, so a `Mi` request is the same size as the memory it is checked against. +- [Azure Functions] An HTTPS invocation waits for `runtime_timeout`, and its metadata request is a POST so the body is delivered. An invoke that returns an HTTP error other than 429 raises. Waiting for runtime metadata with the pub/sub trigger times out after `runtime_timeout`. `runtime_memory` is honoured after the first deploy: each Flex instance memory size gets its own function app, so existing runtimes are deployed again once per size. `lithops runtime update` and `delete` find a runtime by its name. - [Azure Container Apps] `lithops runtime update` and `delete` find a runtime by its image, and `list_runtimes` reports memory as a number. -- [Aliyun FC] Retries while reading runtime metadata resend the payload instead of an empty body. +- [Aliyun FC] The synchronous metadata call waits for `runtime_timeout`. An asynchronous invoke only waits to be accepted. Retries while reading runtime metadata resend the payload instead of an empty body. - [Oracle Functions] Calls run on the function of their own runtime and memory, not on the function of the job started last. Functions and applications past the first page are found by `lithops runtime list`, `delete` and `clean`. - [GCP Functions] [GCP Cloud Run] An HTTPS invocation waits for `runtime_timeout` instead of giving up after 120 seconds, which is while the call is still running. Functions and services past the first page are found by `lithops runtime list` and `clean`. - [Knative] Calls longer than 5 minutes are no longer cut off by Knative, a load balancer ingress is reached at its own port and may be a hostname, `https` service URLs work, a failed call is reported instead of being invoked again for ever, and a service that never gets ready raises an error. diff --git a/lithops/serverless/backends/aliyun_fc/aliyun_fc.py b/lithops/serverless/backends/aliyun_fc/aliyun_fc.py index e4158697b..f134d7100 100644 --- a/lithops/serverless/backends/aliyun_fc/aliyun_fc.py +++ b/lithops/serverless/backends/aliyun_fc/aliyun_fc.py @@ -47,7 +47,10 @@ CONTAINER_MAX_WAIT = 900 CONTAINER_METADATA_MAX_ATTEMPTS = 60 CONTAINER_METADATA_RETRY_SLEEP = 10 -FC_INVOKE_RUNTIME = RuntimeOptions(read_timeout=120000, connect_timeout=30000) +# An async invoke only waits for the platform to accept the call. The +# synchronous metadata call waits for the function itself. +FC_CONNECT_TIMEOUT = 30000 +FC_SUBMIT_TIMEOUT = 120000 def _init_fc3_client(access_key_id, access_key_secret, region, endpoint): @@ -511,6 +514,18 @@ def list_runtimes(self, runtime_name='all'): runtimes.append((img_name, memory, version, function.function_name)) return runtimes + def _submit_runtime(self): + """How long to wait for an asynchronous invoke to be accepted""" + return RuntimeOptions(read_timeout=FC_SUBMIT_TIMEOUT, connect_timeout=FC_CONNECT_TIMEOUT) + + def _metadata_runtime(self): + """The metadata call runs the function, so it waits out runtime_timeout""" + runtime_timeout = (getattr(self, 'config', None) or {}).get('runtime_timeout', 300) + return RuntimeOptions( + read_timeout=(runtime_timeout + 30) * 1000, + connect_timeout=FC_CONNECT_TIMEOUT, + ) + def invoke(self, runtime_name, memory, payload=None): if payload is None: payload = {} @@ -523,7 +538,7 @@ def invoke(self, runtime_name, memory, payload=None): request = fc_models.InvokeFunctionRequest(body=io.BytesIO(payload_bytes)) res = self.fc_client.invoke_function_with_options( - function_name, request, headers, FC_INVOKE_RUNTIME + function_name, request, headers, self._submit_runtime() ) request_id = _get_response_header(res, 'x-fc-request-id') @@ -554,7 +569,7 @@ def _generate_runtime_meta(self, function_name, max_attempts, retry_sleep): # The body stream is consumed by each attempt request = fc_models.InvokeFunctionRequest(body=io.BytesIO(payload_bytes)) res = self.fc_client.invoke_function_with_options( - function_name, request, headers, FC_INVOKE_RUNTIME + function_name, request, headers, self._metadata_runtime() ) runtime_meta = json.loads(_read_invoke_body(res)) if isinstance(runtime_meta, dict) and runtime_meta.get('errorMessage'): diff --git a/lithops/serverless/backends/azure_functions/azure_functions.py b/lithops/serverless/backends/azure_functions/azure_functions.py index 7f628bcf6..d5d90e833 100644 --- a/lithops/serverless/backends/azure_functions/azure_functions.py +++ b/lithops/serverless/backends/azure_functions/azure_functions.py @@ -424,11 +424,17 @@ def invoke(self, runtime_name, memory=None, payload={}, return_result=False): endpoint = f"https://{function_name}.azurewebsites.net" parsed_url = urlparse(endpoint) ctx = ssl._create_unverified_context() - conn = http.client.HTTPSConnection(parsed_url.netloc, context=ctx) + # The handler runs the call inside this request, so the wait is + # the function's own timeout, plus a moment for the response + conn = http.client.HTTPSConnection( + parsed_url.netloc, context=ctx, + timeout=self.af_config.get('runtime_timeout', 300) + 30, + ) route = "/api/lithops_handler" + body = json.dumps(payload, default=str) if return_result: - conn.request("GET", route, body=json.dumps(payload, default=str)) + conn.request("POST", route, body=body) resp = conn.getresponse() resp_text = resp.read().decode("utf-8") conn.close() @@ -440,7 +446,7 @@ def invoke(self, runtime_name, memory=None, payload={}, return_result=False): raise Exception(f'Unable to load runtime metadata: {resp_text}') else: # logger.debug('Invoking calls {}'.format(', '.join(payload['call_ids']))) - conn.request("POST", route, body=json.dumps(payload, default=str)) + conn.request("POST", route, body=body) resp = conn.getresponse() resp_text = resp.read().decode("utf-8") conn.close() diff --git a/lithops/serverless/backends/k8s/k8s.py b/lithops/serverless/backends/k8s/k8s.py index f7746b646..876b04fb9 100644 --- a/lithops/serverless/backends/k8s/k8s.py +++ b/lithops/serverless/backends/k8s/k8s.py @@ -585,16 +585,17 @@ def _start_master(self, docker_image_name): ) def convert_memory_units(self, mem_num, mem_uni): + """ + The value in megabytes. Ki/Mi/Gi use 1024 and K/M/G use 1000, so a + pod request and the memory it is compared with are the same size + """ mem_num = int(mem_num) - - if 'i' in mem_uni: - mem_num *= 1024 - mem_uni = mem_uni[:-1] - if 'K' in mem_uni: - mem_num = mem_num / (1024 if 'i' in mem_uni else 1000) - elif 'G' in mem_uni: - mem_num = mem_num * (1024 if 'i' in mem_uni else 1000) - + factor = 1024 if 'i' in mem_uni.lower() else 1000 + unit = mem_uni.lower() + if 'k' in unit: + mem_num = mem_num / factor + elif 'g' in unit: + mem_num = mem_num * factor return mem_num, 'M' # Detect if granularity, memory or runtime image changed or not diff --git a/lithops/tests/test_azure_aliyun_oracle_backends.py b/lithops/tests/test_azure_aliyun_oracle_backends.py index e90558155..b445f11c8 100644 --- a/lithops/tests/test_azure_aliyun_oracle_backends.py +++ b/lithops/tests/test_azure_aliyun_oracle_backends.py @@ -59,11 +59,12 @@ class FakeHTTPSConnection: reason = 'OK' body = 'invocation-id' - def __init__(self, netloc, context=None): - pass + def __init__(self, netloc, context=None, timeout=None): + self.timeout = timeout def request(self, method, route, body=None): - pass + self.method = method + self.sent = body def getresponse(self): return types.SimpleNamespace( @@ -84,6 +85,25 @@ def test_azure_functions_https_invoke_error_raises(monkeypatch, status, reason): backend.invoke('rt', 2048, {'call_ids': ['00000']}) +def test_azure_functions_https_waits_for_the_runtime_and_posts_metadata(monkeypatch): + af, backend = _azure_functions_backend(trigger='https') + backend.af_config['runtime_timeout'] = 300 + created = {} + + class Recording(FakeHTTPSConnection): + body = '{"preinstalls": []}' + + def __init__(self, netloc, context=None, timeout=None): + super().__init__(netloc, context=context, timeout=timeout) + created['conn'] = self + + monkeypatch.setattr(af.http.client, 'HTTPSConnection', Recording) + assert backend.invoke('rt', 2048, {'get_metadata': True}, return_result=True) == {'preinstalls': []} + assert created['conn'].timeout == 330 + assert created['conn'].method == 'POST' + assert json.loads(created['conn'].sent)['get_metadata'] is True + + def test_azure_functions_https_invoke_success_and_throttle(monkeypatch): af, backend = _azure_functions_backend(trigger='https') monkeypatch.setattr(af, 'time', FakeClock()) @@ -250,14 +270,19 @@ def test_aliyun_fc_metadata_retry_resends_payload(monkeypatch): bodies = [] meta = {'preinstalls': [], 'python_version': '3.12'} + runtimes = [] + def invoke_function_with_options(function_name, request, headers, runtime): + runtimes.append(runtime) bodies.append(request.body.read()) if len(bodies) == 1: raise aliyun_fc.FCClientException(status_code=412, code='PreconditionFailed', message='pending') return types.SimpleNamespace(body=json.dumps(meta).encode('utf-8'), headers={}) backend.fc_client = types.SimpleNamespace(invoke_function_with_options=invoke_function_with_options) + backend.config = {'runtime_timeout': 300} assert backend._generate_runtime_meta('lithops-worker-fn', max_attempts=3, retry_sleep=1) == meta + assert runtimes[0].read_timeout == 330000 assert len(bodies) == 2 assert bodies[0] == bodies[1] assert json.loads(bodies[1])['get_metadata'] is True diff --git a/lithops/tests/test_k8s_backend.py b/lithops/tests/test_k8s_backend.py index 585278d87..4cc887d03 100644 --- a/lithops/tests/test_k8s_backend.py +++ b/lithops/tests/test_k8s_backend.py @@ -18,6 +18,16 @@ from lithops.serverless.backends.k8s.k8s import KubernetesBackend +def test_memory_is_compared_in_megabytes(): + """Mi used to be multiplied by 1024 and then labelled M, so 512Mi was not 512""" + backend = _make_backend() + assert backend.convert_memory_units(512, 'Mi') == (512, 'M') + assert backend.convert_memory_units(512, 'M') == (512, 'M') + assert backend.convert_memory_units(16384, 'Ki') == (16, 'M') + assert backend.convert_memory_units(2, 'Gi') == (2048, 'M') + assert backend.convert_memory_units(2, 'G') == (2000, 'M') + + def _make_backend(overrides=None): """Build a KubernetesBackend without running __init__ (which needs a cluster).""" backend = KubernetesBackend.__new__(KubernetesBackend) From 08c26d6acefe7f0f74bec7f42c0a3a36f502e3af Mon Sep 17 00:00:00 2001 From: JosepSampe Date: Mon, 26 Oct 2026 16:46:37 +0100 Subject: [PATCH 08/13] Fix podman login --- CHANGELOG.md | 1 + lithops/tests/test_utils.py | 21 +++++++++++++++++++++ lithops/utils.py | 6 +++--- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a6ebf9180..d6aab4745 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - [Knative] Updated to current Knative Serving (tested with 1.23): one container per call in flight with the current autoscaling annotations, `kubecfg_context`, `namespace` and `runtime_arch` keys, the Contour networking layer, cluster-local calls from inside the cluster, a deploy that updates an existing service and reports why its revision fails, and a default runtime that runs as a non-root user. ### Fixed +- [Runtime] Podman logins include the Docker Hub registry explicitly, so runtime builds can authenticate without a default registry in `registries.conf`. - [Core] `get_result()` with no futures returns `[]`. `call_async()` chains a list or tuple of futures as one call. `RetryingFunctionExecutor.wait()` shares one timeout across retry rounds and, when it expires, leaves the calls running instead of cancelling them. - [Core] Chaining, cleanup, invocation errors, generator inputs, empty `fs`, `wait()` with `ANY_COMPLETED`, argument binding, and `RetryingFunctionExecutor` retries and percentage waits; a failed object, or chunk of one, is retried on its own. - [Worker] Partitioned object streams (lost, duplicated or cut rows, rows longer than 128 KB, multi-byte `obj_newline`, `readline()`, iteration), module discovery of nested functions, a parameter named `future`, and `extra_env` restoring the previous environment. diff --git a/lithops/tests/test_utils.py b/lithops/tests/test_utils.py index 476b36821..afa0ab691 100644 --- a/lithops/tests/test_utils.py +++ b/lithops/tests/test_utils.py @@ -575,6 +575,27 @@ def test_docker_login_requires_credentials(self): with pytest.raises(Exception, match='docker_server is required'): docker_login('u', 'p', '') + @pytest.mark.parametrize('podman,server,expected_server', [ + (True, 'docker.io', 'docker.io'), + (False, 'docker.io', None), + (True, 'eu-madrid-1.ocir.io', 'eu-madrid-1.ocir.io'), + (False, 'eu-madrid-1.ocir.io', 'eu-madrid-1.ocir.io'), + (False, 'docker.io.example.org', 'docker.io.example.org'), + ]) + def test_docker_login_registry(self, monkeypatch, podman, server, expected_server): + executable = '/usr/bin/podman' if podman else '/usr/bin/docker' + monkeypatch.setattr(lithops_utils, 'get_docker_path', lambda: executable) + monkeypatch.setattr(lithops_utils, 'is_podman', lambda path: podman) + run = MagicMock() + monkeypatch.setattr(lithops_utils, 'run_command', run) + + docker_login('alice', ' secret-token\n', server) + + expected = f'{executable} login -u alice --password-stdin' + if expected_server: + expected += f' {expected_server}' + run.assert_called_once_with(expected, input='secret-token') + def test_get_default_container_name_variants(self): cfg = {'docker_server': 'docker.io', 'docker_user': 'alice'} name = get_default_container_name('openwhisk', cfg, 'lithops') diff --git a/lithops/utils.py b/lithops/utils.py index 36e833ef5..4ca08503a 100644 --- a/lithops/utils.py +++ b/lithops/utils.py @@ -1004,8 +1004,8 @@ def docker_login(docker_user, docker_password, docker_server): """ Log in to a container registry using docker/podman. - Docker Hub must be logged in without an explicit server host, matching - `docker login -u USER --password-stdin`. + Docker Hub uses Docker's default login host. Podman needs an explicit + registry even for Docker Hub when no default registry is configured. """ if not docker_user or not docker_password: raise Exception('docker_user and docker_password are required') @@ -1015,7 +1015,7 @@ def docker_login(docker_user, docker_password, docker_server): docker_path = get_docker_path() docker_password = docker_password.strip() - if 'docker.io' in docker_server: + if docker_server == 'docker.io' and not is_podman(docker_path): cmd = f'{docker_path} login -u {docker_user} --password-stdin' else: cmd = (f'{docker_path} login -u {docker_user} --password-stdin ' From 0318637e485fb3728db4992eaba3270bb51edba1 Mon Sep 17 00:00:00 2001 From: JosepSampe Date: Tue, 27 Oct 2026 12:36:51 +0100 Subject: [PATCH 09/13] Fix Azure backends --- CHANGELOG.md | 7 +- config/config_template.yaml | 2 +- docs/source/compute_config/azure_functions.md | 2 +- docs/source/compute_config/azure_vms.md | 2 +- .../azure_containers/azure_containers.py | 117 ++++++++---- .../azure_functions/azure_functions.py | 40 ++-- .../backends/azure_functions/config.py | 5 +- .../backends/azure_vms/azure_vms.py | 98 +++++----- .../test_azure_aliyun_oracle_backends.py | 177 ++++++++++++++++++ lithops/tests/test_standalone.py | 65 +++++++ 10 files changed, 406 insertions(+), 109 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d6aab4745..a652c3ccf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## [v3.8.1.dev0] ### Changed +- [Azure Functions] Default runtime memory is now 512 MB. - [Core] A storage or monitoring service at a loopback address is refused for serverless workers, which cannot reach it; standalone and Singularity workers get a warning. - [Core] `wait()` timeouts no longer arm a SIGALRM: they are checked between polls, work from any thread and leave the caller's signal handler and alarm alone. - [Core] An `obj_chunk_size` no longer than `obj_newline` is raised to one byte more. @@ -12,6 +13,7 @@ - [Knative] Updated to current Knative Serving (tested with 1.23): one container per call in flight with the current autoscaling annotations, `kubecfg_context`, `namespace` and `runtime_arch` keys, the Contour networking layer, cluster-local calls from inside the cluster, a deploy that updates an existing service and reports why its revision fails, and a default runtime that runs as a non-root user. ### Fixed +- [Azure VMs] Fixed network request serialization with newer SDKs and enabled internet access for workers on newly created subnets. - [Runtime] Podman logins include the Docker Hub registry explicitly, so runtime builds can authenticate without a default registry in `registries.conf`. - [Core] `get_result()` with no futures returns `[]`. `call_async()` chains a list or tuple of futures as one call. `RetryingFunctionExecutor.wait()` shares one timeout across retry rounds and, when it expires, leaves the calls running instead of cancelling them. - [Core] Chaining, cleanup, invocation errors, generator inputs, empty `fs`, `wait()` with `ANY_COMPLETED`, argument binding, and `RetryingFunctionExecutor` retries and percentage waits; a failed object, or chunk of one, is retried on its own. @@ -32,13 +34,14 @@ - [Code Engine] Projects, job definitions and config maps past the first page are listed. `clear()` deletes every job run and config map of the executor, a leftover config map is replaced instead of failing every deploy, and waiting for a job run or a job definition deletion times out. - [Singularity] Runtime metadata that never arrives raises the intended timeout error. - [Kubernetes] Worker memory is compared in megabytes, so a `Mi` request is the same size as the memory it is checked against. -- [Azure Functions] An HTTPS invocation waits for `runtime_timeout`, and its metadata request is a POST so the body is delivered. An invoke that returns an HTTP error other than 429 raises. Waiting for runtime metadata with the pub/sub trigger times out after `runtime_timeout`. `runtime_memory` is honoured after the first deploy: each Flex instance memory size gets its own function app, so existing runtimes are deployed again once per size. `lithops runtime update` and `delete` find a runtime by its name. -- [Azure Container Apps] `lithops runtime update` and `delete` find a runtime by its image, and `list_runtimes` reports memory as a number. +- [Azure Functions] Fixed app lookup and CLI error reporting, invocation and metadata timeouts, metadata POST requests, per-memory deployments, and runtime lookup by name. +- [Azure Container Apps] Added deployment progress logs and cleanup on interrupted creation. Fixed authentication error handling, runtime lookup by image, and numeric memory values. - [Aliyun FC] The synchronous metadata call waits for `runtime_timeout`. An asynchronous invoke only waits to be accepted. Retries while reading runtime metadata resend the payload instead of an empty body. - [Oracle Functions] Calls run on the function of their own runtime and memory, not on the function of the job started last. Functions and applications past the first page are found by `lithops runtime list`, `delete` and `clean`. - [GCP Functions] [GCP Cloud Run] An HTTPS invocation waits for `runtime_timeout` instead of giving up after 120 seconds, which is while the call is still running. Functions and services past the first page are found by `lithops runtime list` and `clean`. - [Knative] Calls longer than 5 minutes are no longer cut off by Knative, a load balancer ingress is reached at its own port and may be a hostname, `https` service URLs work, a failed call is reported instead of being invoked again for ever, and a service that never gets ready raises an error. + ## [v3.8.0] ### Added diff --git a/config/config_template.yaml b/config/config_template.yaml index 4679bd31e..218e3aac0 100644 --- a/config/config_template.yaml +++ b/config/config_template.yaml @@ -211,7 +211,7 @@ #resource_group: # Falls back to azure.resource_group #region: # Falls back to azure.region #runtime: - #runtime_memory: 1536 # MB + #runtime_memory: 512 # MB. Allowed: 512, 2048, 4096; CPU scales with memory #runtime_timeout: 300 # Seconds #functions_version: 4 #trigger: pub/sub # One of: pub/sub, https diff --git a/docs/source/compute_config/azure_functions.md b/docs/source/compute_config/azure_functions.md index 5dff442df..3027127df 100644 --- a/docs/source/compute_config/azure_functions.md +++ b/docs/source/compute_config/azure_functions.md @@ -87,7 +87,7 @@ az login |---|---|---|---|---| |azure_functions| resource_group | |no | Name of a resource group, for example: `LithopsResourceGroup`. Lithops will use the `resource_group` set under the `azure` section if it is not set here | |azure_functions| region | |no | The Flex Consumption plan location for the runtime. Use `az functionapp list-flexconsumption-locations` to view the available locations. For example: `westeurope`, `westus2`, etc. Lithops will use the `region` set under the `azure` section if it is not set here| -|azure_functions | runtime_memory | 2048 | no | Flex Consumption instance memory in MB. Supported values: `512`, `2048`, `4096`. Other values are mapped to the nearest supported size. Each instance memory size is deployed as a separate function app | +|azure_functions | runtime_memory | 512 | no | Flex Consumption instance memory in MB. The default 512 MB receives approximately 0.25 CPU automatically. Supported values: `512`, `2048`, `4096`. Other values are mapped to the nearest supported size. Each instance memory size is deployed as a separate function app | |azure_functions | max_workers | 1000 | no | Max number of parallel workers. Although Azure limits the number of workers to 200, it is convenient to keep this value high| |azure_functions | worker_processes | 1 | no | Lithops-side parallelism setting. Not applied as an Azure Functions app setting on Flex Consumption | |azure_functions| runtime | |no | Runtime name already deployed in the service| diff --git a/docs/source/compute_config/azure_vms.md b/docs/source/compute_config/azure_vms.md index 36191f414..0462e1e84 100644 --- a/docs/source/compute_config/azure_vms.md +++ b/docs/source/compute_config/azure_vms.md @@ -206,7 +206,7 @@ lithops clean -b azure_vms -s azure_storage --all ## Architecture diagram -Lithops creates a **Virtual Network** and **subnet** in your resource group (unless you provide existing ones). The master VM gets a **static public IP**; worker VMs use **private IPs only** and reach the internet through Azure's default subnet outbound routing. +Lithops creates a **Virtual Network** and **subnet** in your resource group (unless you provide existing ones). The master VM gets a **static public IP**; worker VMs use **private IPs only**. Lithops explicitly enables default outbound access on subnets it creates so workers can download dependencies. User-provided subnets must supply outbound connectivity, such as a NAT gateway. ```mermaid flowchart TB diff --git a/lithops/serverless/backends/azure_containers/azure_containers.py b/lithops/serverless/backends/azure_containers/azure_containers.py index 937504825..905f7f82e 100644 --- a/lithops/serverless/backends/azure_containers/azure_containers.py +++ b/lithops/serverless/backends/azure_containers/azure_containers.py @@ -21,6 +21,7 @@ import logging import hashlib import shutil +import threading import subprocess as sp from azure.storage.queue import QueueServiceClient @@ -118,6 +119,23 @@ def _run_az_command(self, cmd, return_json=False, return_result=False): ) from e return result.replace('"', '') + def _run_deployment_command(self, cmd): + finished = threading.Event() + + def report_progress(): + elapsed = 0 + while not finished.wait(60): + elapsed += 60 + logger.debug(f'Still deploying Azure Container App ({elapsed}s elapsed)') + + progress = threading.Thread(target=report_progress, daemon=True) + progress.start() + try: + return self._run_az_command(cmd) + finally: + finished.set() + progress.join() + def _format_containerapp_name(self, runtime_name, runtime_memory, version=__version__): """ Formates the conatiner app name @@ -158,8 +176,10 @@ def _get_containerapp_provisioning_state(self, containerapp_name): try: state = self._run_az_command(cmd, return_result=True) return (state or '').strip() or None - except Exception: - return None + except Exception as exc: + if 'ResourceNotFound' in str(exc): + return None + raise def _wait_until_containerapp_idle(self, containerapp_name, timeout=360): """ @@ -328,47 +348,64 @@ def _create_app(self, runtime_name, memory, timeout): last_error = None cmd = None - while retries < 10: - try: - state = self._wait_until_containerapp_idle(containerapp_name) - if state and state.lower() == 'failed': - logger.warning( - f'Container app {containerapp_name} is in Failed state, recreating' - ) - del_cmd = ( + created = False + try: + while retries < 10: + try: + state = self._wait_until_containerapp_idle(containerapp_name) + if state and state.lower() == 'failed': + logger.warning( + f'Container app {containerapp_name} is in Failed state, recreating' + ) + del_cmd = ( + f'az containerapp delete --name {containerapp_name} ' + f'--resource-group {self.resource_group} -y -o none' + ) + self._run_az_command(del_cmd) + self._wait_until_containerapp_idle(containerapp_name) + state = None + + if state: + logger.debug(f'Container app {containerapp_name} already exists, updating') + cmd = (f'az containerapp update --name {containerapp_name} ' + f'--resource-group {self.resource_group} ' + f'--yaml {config.CA_JSON_LOCATION} -o none') + else: + created = True + cmd = (f'az containerapp create --name {containerapp_name} ' + f'--resource-group {self.resource_group} ' + f'--yaml {config.CA_JSON_LOCATION} -o none') + self._run_deployment_command(cmd) + os.remove(config.CA_JSON_LOCATION) + deployed = True + break + except Exception as e: + if any(code in str(e) for code in ENV_NOT_FOUND_ERRORS): + raise Exception(self._environment_not_found_msg()) from e + if any(code in str(e) for code in NON_RETRYABLE_DEPLOY_ERRORS): + raise Exception(f'The Azure Container App cannot be deployed: {e}') from e + last_error = e + in_progress = 'ContainerAppOperationInProgress' in str(e) + logger.warning(f'Container app deploy attempt {retries + 1} failed: {e}') + if in_progress: + self._wait_until_containerapp_idle(containerapp_name) + else: + time.sleep(15) + retries += 1 + + except KeyboardInterrupt: + if created: + logger.info(f'Deployment interrupted; deleting Container App {containerapp_name}') + try: + self._run_az_command( f'az containerapp delete --name {containerapp_name} ' - f'--resource-group {self.resource_group} -y -o none' + f'--resource-group {self.resource_group} --yes --no-wait -o none' ) - self._run_az_command(del_cmd) - self._wait_until_containerapp_idle(containerapp_name) - state = None - - if state: - logger.debug(f'Container app {containerapp_name} already exists, updating') - cmd = (f'az containerapp update --name {containerapp_name} ' - f'--resource-group {self.resource_group} ' - f'--yaml {config.CA_JSON_LOCATION} -o none') - else: - cmd = (f'az containerapp create --name {containerapp_name} ' - f'--resource-group {self.resource_group} ' - f'--yaml {config.CA_JSON_LOCATION} -o none') - self._run_az_command(cmd) + except Exception as exc: + logger.warning(f'Could not delete interrupted deployment {containerapp_name}: {exc}') + if os.path.exists(config.CA_JSON_LOCATION): os.remove(config.CA_JSON_LOCATION) - deployed = True - break - except Exception as e: - if any(code in str(e) for code in ENV_NOT_FOUND_ERRORS): - raise Exception(self._environment_not_found_msg()) from e - if any(code in str(e) for code in NON_RETRYABLE_DEPLOY_ERRORS): - raise Exception(f'The Azure Container App cannot be deployed: {e}') from e - last_error = e - in_progress = 'ContainerAppOperationInProgress' in str(e) - logger.warning(f'Container app deploy attempt {retries + 1} failed: {e}') - if in_progress: - self._wait_until_containerapp_idle(containerapp_name) - else: - time.sleep(15) - retries += 1 + raise if not deployed: raise Exception(f"The Azure Container App cannot be deployed: {cmd}") from last_error diff --git a/lithops/serverless/backends/azure_functions/azure_functions.py b/lithops/serverless/backends/azure_functions/azure_functions.py index d5d90e833..d4fe0eb24 100644 --- a/lithops/serverless/backends/azure_functions/azure_functions.py +++ b/lithops/serverless/backends/azure_functions/azure_functions.py @@ -77,8 +77,6 @@ def _run_az_command(self, cmd, return_json=False, return_result=False): stderr cannot fill a pipe and deadlock (unlike check_call + PIPE). """ self._check_az_cli() - debug = logger.getEffectiveLevel() == logging.DEBUG - capture = return_json or return_result or not debug logger.debug(f'Running Azure CLI: {cmd}') try: @@ -86,8 +84,8 @@ def _run_az_command(self, cmd, return_json=False, return_result=False): cmd, shell=True, encoding='UTF-8', - stdout=sp.PIPE if capture else None, - stderr=sp.PIPE if capture else None, + stdout=sp.PIPE, + stderr=sp.PIPE, stdin=sp.DEVNULL, check=True, ) @@ -115,17 +113,29 @@ def _function_app_exists(self, function_name): cmd = (f'az functionapp show --name {function_name} ' f'--resource-group {self.resource_group}') try: - # Probe only; hide az CLI's ResourceNotFound when the app is missing. - sp.check_call( - cmd, - shell=True, - stdout=sp.DEVNULL, - stderr=sp.DEVNULL, - ) + self._run_az_command(cmd) return True - except sp.CalledProcessError: - logger.debug(f'Function app {function_name} not found, will create it') - return False + except Exception as exc: + # Only a missing app permits creation. Authentication, authorization + # and transport failures must retain the CLI's original diagnostic. + cause = exc.__cause__ + if isinstance(cause, sp.CalledProcessError): + detail = (cause.stderr or cause.stdout or '').strip() + missing = detail.startswith(('ERROR: (ResourceNotFound)', '(ResourceNotFound)')) + # Some CLI versions wrap the ARM JSON error in Not Found(...). + prefix = 'ERROR: Not Found(' + if detail.startswith(prefix): + try: + error, _ = json.JSONDecoder().raw_decode(detail[len(prefix):]) + except ValueError: + pass + else: + if isinstance(error, dict) and isinstance(error.get('error'), dict): + missing = error['error'].get('code') == 'ResourceNotFound' + if missing: + logger.debug(f'Function app {function_name} not found, will create it') + return False + raise def _create_deploy_zip(self, build_dir): """ @@ -543,7 +553,7 @@ def list_runtimes(self, runtime_name='all'): and functionapp['Tags']['type'] == 'lithops-runtime': version = functionapp['Tags']['lithops_version'] name = functionapp['Tags']['runtime_name'] - memory = int(functionapp['Tags'].get('runtime_memory', config.DEFAULT_CONFIG_KEYS['runtime_memory'])) + memory = int(functionapp['Tags'].get('runtime_memory', 512)) if runtime_name == name or runtime_name == 'all': runtimes.append((name, memory, version, functionapp['Name'])) diff --git a/lithops/serverless/backends/azure_functions/config.py b/lithops/serverless/backends/azure_functions/config.py index eb9bf1c70..122ddd347 100644 --- a/lithops/serverless/backends/azure_functions/config.py +++ b/lithops/serverless/backends/azure_functions/config.py @@ -26,8 +26,8 @@ FH_ZIP_LOCATION = os.path.join(os.getcwd(), 'lithops_azure_fa.zip') DEFAULT_CONFIG_KEYS = { - 'runtime_timeout': 300, # Default: 300 seconds => 5 minutes - 'runtime_memory': 2048, # Default Flex Consumption instance memory: 2048 MB + 'runtime_timeout': 300, + 'runtime_memory': 512, # Flex Consumption assigns approximately 0.25 CPU at 512 MB 'max_workers': 1000, 'worker_processes': 1, 'invoke_pool_threads': 10, @@ -151,6 +151,7 @@ def get_flex_instance_memory(memory): requests PyYAML kubernetes +python-dateutil>=2.9.0,<3 cloudpickle ps-mem tblib diff --git a/lithops/standalone/backends/azure_vms/azure_vms.py b/lithops/standalone/backends/azure_vms/azure_vms.py index 2b5feacaa..16fcb4870 100644 --- a/lithops/standalone/backends/azure_vms/azure_vms.py +++ b/lithops/standalone/backends/azure_vms/azure_vms.py @@ -38,6 +38,10 @@ VirtualMachine, ) from azure.mgmt.network import NetworkManagementClient +from azure.mgmt.network.models import ( + AddressSpace, NetworkInterface, NetworkInterfaceIPConfiguration, NetworkSecurityGroup, + PublicIPAddress, PublicIPAddressSku, SecurityRule, Subnet, VirtualNetwork, +) from azure.core.exceptions import HttpResponseError, ResourceNotFoundError from lithops.version import __version__ @@ -225,10 +229,10 @@ def _create_vnet(self): poller = self.network_client.virtual_networks.begin_create_or_update( self.config['resource_group'], self.vnet_name, - { - "location": self.location, - "address_space": {"address_prefixes": ["10.0.0.0/16"]}, - }, + VirtualNetwork( + location=self.location, + address_space=AddressSpace(address_prefixes=["10.0.0.0/16"]), + ), ) vnet_result = poller.result() self.config['vnet_id'] = vnet_result.id @@ -268,7 +272,7 @@ def _create_subnet(self): self.config['resource_group'], self.vnet_name, self.subnet_name, - {"address_prefix": "10.0.0.0/24"}, + Subnet(address_prefix="10.0.0.0/24", default_outbound_access=True), ) subnet_result = poller.result() self.config['subnet_id'] = subnet_result.id @@ -330,10 +334,10 @@ def _create_security_group(self): { "name": "allow-ssh", "protocol": "Tcp", - "sourcePortRange": "*", - "destinationPortRange": "22", - "sourceAddressPrefix": "*", - "destinationAddressPrefix": "*", + "source_port_range": "*", + "destination_port_range": "22", + "source_address_prefix": "*", + "destination_address_prefix": "*", "access": "Allow", "direction": "Inbound", "priority": 100 @@ -341,10 +345,10 @@ def _create_security_group(self): { "name": "allow-master-port-8080", "protocol": "Tcp", - "sourcePortRange": "*", - "destinationPortRange": "8080", - "sourceAddressPrefix": "10.0.0.0/24", - "destinationAddressPrefix": "*", + "source_port_range": "*", + "destination_port_range": "8080", + "source_address_prefix": "10.0.0.0/24", + "destination_address_prefix": "*", "access": "Allow", "direction": "Inbound", "priority": 101 @@ -352,10 +356,10 @@ def _create_security_group(self): { "name": "allow-worker-port-8081", "protocol": "Tcp", - "sourcePortRange": "*", - "destinationPortRange": "8081", - "sourceAddressPrefix": "10.0.0.0/24", - "destinationAddressPrefix": "*", + "source_port_range": "*", + "destination_port_range": "8081", + "source_address_prefix": "10.0.0.0/24", + "destination_address_prefix": "*", "access": "Allow", "direction": "Inbound", "priority": 102 @@ -363,10 +367,10 @@ def _create_security_group(self): { "name": "allow-redis-port-6379", "protocol": "Tcp", - "sourcePortRange": "*", - "destinationPortRange": "6379", - "sourceAddressPrefix": "10.0.0.0/24", - "destinationAddressPrefix": "*", + "source_port_range": "*", + "destination_port_range": "6379", + "source_address_prefix": "10.0.0.0/24", + "destination_address_prefix": "*", "access": "Allow", "direction": "Inbound", "priority": 103 @@ -374,10 +378,10 @@ def _create_security_group(self): ] # Define the network security group to contain the rule - network_security_group = { - "location": self.location, - "securityRules": nsg_rules - } + network_security_group = NetworkSecurityGroup( + location=self.location, + security_rules=[SecurityRule(**rule) for rule in nsg_rules], + ) # Create or update the network security group poller = self.network_client.network_security_groups.begin_create_or_update( @@ -423,12 +427,12 @@ def get_floating_ip(fip_name): poller = self.network_client.public_ip_addresses.begin_create_or_update( self.config['resource_group'], floating_ip_name, - { - "location": self.location, - "sku": {"name": "Standard"}, - "public_ip_allocation_method": "Static", - "public_ip_address_version": "IPV4", - }, + PublicIPAddress( + location=self.location, + sku=PublicIPAddressSku(name="Standard"), + public_ip_allocation_method="Static", + public_ip_address_version="IPv4", + ), ) ip_address_result = poller.result() self.config['floating_ip'] = ip_address_result.ip_address @@ -973,34 +977,34 @@ def _create_instance(self, user_data=None): vm_created = False # Create NIC - nic_params = { - 'location': self.location, - 'ip_configurations': [{ - 'name': 'ipconfig1', - 'subnet': {'id': self.config['subnet_id']}, - }], - "network_security_group": {"id": self.config['security_group_id']} - } + nic_params = NetworkInterface( + location=self.location, + ip_configurations=[NetworkInterfaceIPConfiguration( + name='ipconfig1', + subnet=Subnet(id=self.config['subnet_id']), + )], + network_security_group=NetworkSecurityGroup(id=self.config['security_group_id']), + ) try: if self.public and not self.public_ip: poller = self.network_client.public_ip_addresses.begin_create_or_update( self.config['resource_group'], self.name + '-ip', - { - "location": self.location, - "sku": {"name": "Standard"}, - "public_ip_allocation_method": "Static", - "public_ip_address_version": "IPV4", - }, + PublicIPAddress( + location=self.location, + sku=PublicIPAddressSku(name="Standard"), + public_ip_allocation_method="Static", + public_ip_address_version="IPv4", + ), ) ip_address_result = poller.result() public_ip_created = True self.public_ip = ip_address_result.ip_address - nic_params['ip_configurations'][0]['public_ip_address'] = {"id": ip_address_result.id} + nic_params.ip_configurations[0].public_ip_address = PublicIPAddress(id=ip_address_result.id) elif self.public: - nic_params['ip_configurations'][0]['public_ip_address'] = {"id": self.config['floating_ip_id']} + nic_params.ip_configurations[0].public_ip_address = PublicIPAddress(id=self.config['floating_ip_id']) poller = self.network_client.network_interfaces.begin_create_or_update( self.config['resource_group'], diff --git a/lithops/tests/test_azure_aliyun_oracle_backends.py b/lithops/tests/test_azure_aliyun_oracle_backends.py index b445f11c8..e2f8d90ce 100644 --- a/lithops/tests/test_azure_aliyun_oracle_backends.py +++ b/lithops/tests/test_azure_aliyun_oracle_backends.py @@ -54,6 +54,79 @@ def _azure_functions_backend(trigger='pub/sub'): return af, backend +@pytest.mark.parametrize('detail, missing', [ + ('ERROR: (ResourceNotFound) The resource was not found.', True), + ('ERROR: Not Found({"error":{"code":"ResourceNotFound","message":"App missing"}})', True), + ('ERROR: Not Found({"error":{"code":"ResourceGroupNotFound"}})', False), + ('ERROR: Not Found({"error":{"code":"AuthorizationFailed","message":"ResourceNotFound"}})', False), + ('ERROR: Not Found(invalid JSON)', False), + ('ERROR: Not Found({"error":null})', False), + ('ERROR: Not Found([])', False), + ('AADSTS9002313: Invalid request. Run az login --tenant tenant-id', False), + ('ERROR: (AuthorizationFailed) Access denied', False), + ('ERROR: (ResourceGroupNotFound) Resource group missing', False), + ('Connection failed', False), +]) +@pytest.mark.parametrize('debug', [False, True]) +def test_azure_functions_probe_preserves_cli_errors(monkeypatch, detail, missing, debug): + af, backend = _azure_functions_backend() + monkeypatch.setattr(backend, '_check_az_cli', lambda: None) + monkeypatch.setattr(af.logger, 'getEffectiveLevel', lambda: af.logging.DEBUG if debug else af.logging.INFO) + + def run(cmd, **kwargs): + assert kwargs['stdout'] == af.sp.PIPE + assert kwargs['stderr'] == af.sp.PIPE + raise af.sp.CalledProcessError(1, cmd, stderr=detail) + + monkeypatch.setattr(af.sp, 'run', run) + if missing: + assert backend._function_app_exists('worker') is False + else: + with pytest.raises(Exception, match=detail.split(':')[0].replace('(', r'\(').replace(')', r'\)')): + backend._function_app_exists('worker') + + +def test_azure_functions_probe_existing_app(monkeypatch): + af, backend = _azure_functions_backend() + monkeypatch.setattr(backend, '_check_az_cli', lambda: None) + monkeypatch.setattr(af.sp, 'run', lambda *a, **kw: types.SimpleNamespace(stdout='{}', stderr='')) + assert backend._function_app_exists('worker') is True + + +def test_azure_functions_default_runtime_dateutil_compatibility(monkeypatch, tmp_path): + from packaging.requirements import Requirement + + af, backend = _azure_functions_backend() + monkeypatch.setattr(af, 'TEMP_DIR', str(tmp_path)) + builds = [] + + def build(runtime_name, requirements_file): + with open(requirements_file) as req_file: + requirements = [Requirement(line.strip()) for line in req_file + if line.strip() and not line.startswith('#')] + dateutil = next(req for req in requirements if req.name == 'python-dateutil') + assert '2.9.0' in dateutil.specifier + assert '2.9.0.post0' in dateutil.specifier + assert '3.0.0' not in dateutil.specifier + builds.append(runtime_name) + + monkeypatch.setattr(backend, 'build_runtime', build) + backend._build_default_runtime('default-runtime-v312') + assert builds == ['default-runtime-v312'] + assert not (tmp_path / 'az_default_requirements.txt').exists() + + +def test_azure_functions_default_memory(): + af = _azure_functions_module() + config = { + 'azure_functions': {'resource_group': 'rg', 'region': 'westeurope'}, + 'azure_storage': {'storage_account_name': 'test', 'storage_account_key': 'test'}, + } + af.config.load_config(config) + assert config['azure_functions']['runtime_memory'] == 512 + assert af.config.get_flex_instance_memory(config['azure_functions']['runtime_memory']) == 512 + + class FakeHTTPSConnection: status = 200 reason = 'OK' @@ -232,6 +305,110 @@ def run_az(cmd, return_json=False, **kw): # Azure Container Apps # /---------------------------------------------------------------------------/ +@pytest.mark.parametrize('fails', [False, True]) +def test_azure_containers_deployment_progress(monkeypatch, caplog, fails): + pytest.importorskip('azure.storage.queue') + from lithops.serverless.backends.azure_containers import azure_containers as ac + + backend = ac.AzureContainerAppBackend.__new__(ac.AzureContainerAppBackend) + waits = [] + finished = [] + joined = [] + + class Event: + def wait(self, timeout): + waits.append(timeout) + return len(waits) > 2 + + def set(self): + finished.append(True) + + class Thread: + def __init__(self, target, daemon): + self.target = target + + def start(self): + self.target() + + def join(self): + joined.append(True) + + monkeypatch.setattr(ac.threading, 'Event', Event) + monkeypatch.setattr(ac.threading, 'Thread', Thread) + + def run(cmd): + assert cmd == 'az containerapp create' + if fails: + raise RuntimeError('deployment failed') + return 'done' + + monkeypatch.setattr(backend, '_run_az_command', run) + with caplog.at_level(ac.logging.DEBUG, logger=ac.__name__): + if fails: + with pytest.raises(RuntimeError, match='deployment failed'): + backend._run_deployment_command('az containerapp create') + else: + assert backend._run_deployment_command('az containerapp create') == 'done' + assert waits == [60, 60, 60] + assert 'Still deploying Azure Container App (60s elapsed)' in caplog.text + assert 'Still deploying Azure Container App (120s elapsed)' in caplog.text + assert finished == joined == [True] + + +@pytest.mark.parametrize('state', [None, 'Succeeded']) +@pytest.mark.parametrize('delete_fails', [False, True]) +def test_azure_containers_interrupt_cleans_new_app(monkeypatch, tmp_path, state, delete_fails): + pytest.importorskip('azure.storage.queue') + from lithops.serverless.backends.azure_containers import azure_containers as ac + + backend = ac.AzureContainerAppBackend.__new__(ac.AzureContainerAppBackend) + backend.trigger = 'https' + backend.storage_account_name = 'storage' + backend.resource_group = 'rg' + backend.location = 'eastus' + backend.ac_config = {'max_workers': 1} + manifest = tmp_path / 'app.json' + monkeypatch.setattr(ac.config, 'CA_JSON_LOCATION', str(manifest)) + monkeypatch.setattr(backend, '_get_managed_environment_id', lambda: 'environment-id') + monkeypatch.setattr(backend, '_wait_until_containerapp_idle', lambda name: state) + commands = [] + + def run(cmd, **kwargs): + commands.append(cmd) + if ' delete ' in cmd and delete_fails: + raise RuntimeError('delete failed') + return 'connection-string' + + def deploy(cmd): + assert (' update ' if state else ' create ') in cmd + raise KeyboardInterrupt + + monkeypatch.setattr(backend, '_run_az_command', run) + monkeypatch.setattr(backend, '_run_deployment_command', deploy) + with pytest.raises(KeyboardInterrupt): + backend._create_app('image:latest', 512, 600) + deletes = [cmd for cmd in commands if ' delete ' in cmd] + assert len(deletes) == (0 if state else 1) + if deletes: + assert '--no-wait' in deletes[0] + assert not manifest.exists() + + +def test_azure_containers_probe_preserves_authentication_error(monkeypatch): + pytest.importorskip('azure.storage.queue') + from lithops.serverless.backends.azure_containers import azure_containers as ac + + backend = ac.AzureContainerAppBackend.__new__(ac.AzureContainerAppBackend) + backend.resource_group = 'rg' + + def run(*args, **kwargs): + raise RuntimeError('AuthorizationFailed') + + monkeypatch.setattr(backend, '_run_az_command', run) + with pytest.raises(RuntimeError, match='AuthorizationFailed'): + backend._get_containerapp_provisioning_state('app') + + def test_azure_containers_list_runtimes_by_runtime_name(monkeypatch): pytest.importorskip('azure.storage.queue') from lithops.serverless.backends.azure_containers import azure_containers as ac diff --git a/lithops/tests/test_standalone.py b/lithops/tests/test_standalone.py index 20e01c915..12ec6f524 100644 --- a/lithops/tests/test_standalone.py +++ b/lithops/tests/test_standalone.py @@ -1576,3 +1576,68 @@ def wait(queue_name, exec_mode): assert polls['n'] == 2 assert sa_worker.worker_threads[0]['status'] == WorkerStatus.IDLE.value + + +class TestAzureNetworkSerialization: + """Check the actual SDK serialization, without contacting Azure.""" + + @staticmethod + def _wire(model): + if hasattr(model, 'as_dict'): + return model.as_dict() + return model.serialize() + + def test_network_resources_use_arm_properties(self): + backend, *_ = _azure_backend_for_init() + from azure.core.exceptions import ResourceNotFoundError + backend._init_created = set() + client = backend.network_client + client.subnets.list.return_value = [] + client.network_security_groups.get.side_effect = ResourceNotFoundError('missing') + client.public_ip_addresses.get.side_effect = ResourceNotFoundError('missing') + + backend._create_vnet() + body = self._wire(client.virtual_networks.begin_create_or_update.call_args.args[2]) + assert body['properties']['addressSpace']['addressPrefixes'] == ['10.0.0.0/16'] + assert 'address_space' not in body + + backend._create_subnet() + body = self._wire(client.subnets.begin_create_or_update.call_args.args[3]) + assert body['properties']['addressPrefix'] == '10.0.0.0/24' + assert body['properties']['defaultOutboundAccess'] is True + + backend._create_security_group() + body = self._wire(client.network_security_groups.begin_create_or_update.call_args.args[2]) + rules = body['properties']['securityRules'] + assert len(rules) == 4 + assert rules[0]['properties']['destinationPortRange'] == '22' + assert rules[1]['properties']['sourceAddressPrefix'] == '10.0.0.0/24' + + backend._create_master_floating_ip() + body = self._wire(client.public_ip_addresses.begin_create_or_update.call_args.args[2]) + assert body['sku']['name'] == 'Standard' + assert body['properties']['publicIPAllocationMethod'] == 'Static' + assert body['properties']['publicIPAddressVersion'] == 'IPv4' + + @pytest.mark.parametrize('public', [False, True]) + def test_nic_references_are_serialized(self, tmp_path, monkeypatch, public): + pytest.importorskip('azure.mgmt.network') + from lithops.standalone.backends.azure_vms import azure_vms + captured = [] + model = azure_vms.NetworkInterface + + def capture(**kwargs): + nic = model(**kwargs) + captured.append(nic) + return nic + + monkeypatch.setattr(azure_vms, 'NetworkInterface', capture) + TestAzureSpotWorkers._created_vm(tmp_path, public=public, spot=False) + body = self._wire(captured[0])['properties'] + assert body['networkSecurityGroup']['id'] == 'sg' + ip = body['ipConfigurations'][0]['properties'] + assert ip['subnet']['id'] == 's' + if public: + assert ip['publicIPAddress']['id'] == 'ip' + else: + assert 'publicIPAddress' not in ip From 3f677acebaa23b2a02acbb1591a6be8c498d3533 Mon Sep 17 00:00:00 2001 From: JosepSampe Date: Tue, 27 Oct 2026 16:11:00 +0100 Subject: [PATCH 10/13] Fix GCP runtime list --- CHANGELOG.md | 1 + .../backends/gcp_functions/gcp_functions.py | 10 ++++--- lithops/tests/test_serverless.py | 26 +++++++++++++++++++ 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a652c3ccf..8d00fe7a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ - [Knative] Updated to current Knative Serving (tested with 1.23): one container per call in flight with the current autoscaling annotations, `kubecfg_context`, `namespace` and `runtime_arch` keys, the Contour networking layer, cluster-local calls from inside the cluster, a deploy that updates an existing service and reports why its revision fails, and a default runtime that runs as a non-root user. ### Fixed +- [GCP Functions] Cleaning runtimes works when service configuration is missing. - [Azure VMs] Fixed network request serialization with newer SDKs and enabled internet access for workers on newly created subnets. - [Runtime] Podman logins include the Docker Hub registry explicitly, so runtime builds can authenticate without a default registry in `registries.conf`. - [Core] `get_result()` with no futures returns `[]`. `call_async()` chains a list or tuple of futures as one call. `RetryingFunctionExecutor.wait()` shares one timeout across retry rounds and, when it expires, leaves the calls running instead of cancelling them. diff --git a/lithops/serverless/backends/gcp_functions/gcp_functions.py b/lithops/serverless/backends/gcp_functions/gcp_functions.py index 742b02f67..d47f48beb 100644 --- a/lithops/serverless/backends/gcp_functions/gcp_functions.py +++ b/lithops/serverless/backends/gcp_functions/gcp_functions.py @@ -373,7 +373,8 @@ def delete_runtime(self, runtime_name, runtime_memory, version=__version__, function_name = self._format_function_name( runtime_name, runtime_memory, version) function_location = self._get_function_location(function_name) - logger.info(f'Deleting runtime: {runtime_name} - {runtime_memory}MB') + memory_info = f' - {runtime_memory}MB' if runtime_memory is not None else '' + logger.info(f'Deleting runtime: {runtime_name}{memory_info}') # Delete function try: @@ -433,12 +434,13 @@ def list_runtimes(self, runtime_name='all'): logger.debug('Listing deployed runtimes') runtimes = [] for func in self._list_functions(): - if func['labels'] and 'type' in func['labels'] \ - and func['labels']['type'] == 'lithops-runtime': + labels = func.get('labels') or {} + if labels.get('type') == 'lithops-runtime': fn_name = func['name'].rsplit('/', 1)[-1] version = func['labels']['lithops_version'].replace('-', '.') rt_name = func['labels']['runtime_name'] - memory = self._memory_from_gcfv2(func['serviceConfig']['availableMemory']) + memory_value = (func.get('serviceConfig') or {}).get('availableMemory') + memory = self._memory_from_gcfv2(memory_value) if memory_value is not None else None if runtime_name == rt_name or runtime_name == 'all': runtimes.append((rt_name, memory, version, fn_name)) diff --git a/lithops/tests/test_serverless.py b/lithops/tests/test_serverless.py index c11d1a66a..7ba4a6c5f 100644 --- a/lithops/tests/test_serverless.py +++ b/lithops/tests/test_serverless.py @@ -487,6 +487,32 @@ def fn(name): assert [row[3] for row in backend.list_runtimes()] == ['a', 'b'] assert list_call.call_args_list[1].kwargs['pageToken'] == 'tok' + @pytest.mark.parametrize('service_config', [None, {}, {'availableMemory': '512Mi'}]) + def test_clean_deletes_incomplete_functions(self, service_config): + backend = self._backend() + function = { + 'name': 'projects/p/locations/l/functions/failed-runtime', + 'state': 'UNKNOWN', + 'labels': { + 'type': 'lithops-runtime', + 'lithops_version': '3-8-1-dev0', + 'runtime_name': 'rt', + }, + } + if service_config is not None: + function['serviceConfig'] = service_config + backend._list_functions = MagicMock(return_value=[ + {'name': 'unrelated-function'}, {'labels': None}, function, + ]) + memory = 512 if service_config else None + assert backend.list_runtimes() == [('rt', memory, '3.8.1.dev0', 'failed-runtime')] + assert backend.list_runtimes('other') == [] + backend.delete_runtime = MagicMock() + backend.clean() + backend.delete_runtime.assert_called_once_with( + 'rt', memory, '3.8.1.dev0', function_name='failed-runtime' + ) + class TestCloudRunDeployAndDelete: From 9de3c6bcebaa73ea830aeeabcbd70433f5ee4b31 Mon Sep 17 00:00:00 2001 From: JosepSampe Date: Tue, 27 Oct 2026 17:29:23 +0100 Subject: [PATCH 11/13] Fix gcp backends --- CHANGELOG.md | 2 + config/config_template.yaml | 4 +- docs/source/compute_config/gcp_cloudrun.md | 4 +- .../compute_config/gcp_compute_engine.md | 6 +- docs/source/compute_config/gcp_functions.md | 6 +- .../backends/gcp_cloudrun/cloudrun.py | 2 + .../backends/gcp_functions/gcp_functions.py | 2 + .../backends/gcp_compute_engine/config.py | 27 ++++++-- .../gcp_compute_engine/gcp_compute_engine.py | 18 ++++-- lithops/tests/test_config.py | 61 +++++++++++++++++++ lithops/tests/test_serverless.py | 30 +++++++++ lithops/tests/test_standalone.py | 60 ++++++++++++++++++ 12 files changed, 203 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d00fe7a9..57eeace72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ - [Knative] Updated to current Knative Serving (tested with 1.23): one container per call in flight with the current autoscaling annotations, `kubecfg_context`, `namespace` and `runtime_arch` keys, the Contour networking layer, cluster-local calls from inside the cluster, a deploy that updates an existing service and reports why its revision fails, and a default runtime that runs as a non-root user. ### Fixed +- [GCP Compute Engine] Cleanup tolerates VMs already deleted by another process. +- [GCP] Compute Engine infers the project from credentials when omitted; Functions and Cloud Run honour explicit project IDs. - [GCP Functions] Cleaning runtimes works when service configuration is missing. - [Azure VMs] Fixed network request serialization with newer SDKs and enabled internet access for workers on newly created subnets. - [Runtime] Podman logins include the Docker Hub registry explicitly, so runtime builds can authenticate without a default registry in `registries.conf`. diff --git a/config/config_template.yaml b/config/config_template.yaml index 218e3aac0..c1d70d0f7 100644 --- a/config/config_template.yaml +++ b/config/config_template.yaml @@ -140,7 +140,7 @@ # Google Cloud Platform – credentials shared by all gcp_* sections # ============================================================================= #gcp: - #project_name: + #project_name: # Optional; inferred from credentials when omitted #region: # e.g. us-east1 #credentials_path: @@ -177,7 +177,7 @@ # Google Compute Engine – standalone compute #gcp_compute_engine: - #project_name: # Falls back to gcp.project_name + #project_name: # Falls back to gcp.project_name, then credentials #zone: # e.g. us-east1-b. Mandatory #region: # Falls back to gcp.region; derived from zone if omitted #service_account: diff --git a/docs/source/compute_config/gcp_cloudrun.md b/docs/source/compute_config/gcp_cloudrun.md index bd17658a6..982755fbb 100644 --- a/docs/source/compute_config/gcp_cloudrun.md +++ b/docs/source/compute_config/gcp_cloudrun.md @@ -73,12 +73,14 @@ Replace `gcf-artifacts`, `us-east1`, `lithops-dev`, and the service account emai |Group|Key|Default|Mandatory|Additional info| |---|---|---|---|---| +|gcp | project_name | from credentials |no | GCP project ID | |gcp | region | |yes | Region name of the GCP services (e.g. `us-east1`) | -|gcp | credentials_path | | yes | **Absolute** path of your JSON key file downloaded in step 7 (e.g. `/home/myuser/lithops-invoker1234567890.json`). Alternatively you can set `GOOGLE_APPLICATION_CREDENTIALS` environment variable. If not provided it will try to load the default credentials from the environment| +|gcp | credentials_path | |yes | Service account JSON file, or set `GOOGLE_APPLICATION_CREDENTIALS`. | ### Google Cloud Run |Group|Key|Default|Mandatory|Additional info| |---|---|---|---|---| +|gcp_cloudrun | project_name | from gcp or credentials |no | GCP project ID; overrides `gcp.project_name` | |gcp_cloudrun | region | |no | Region name (e.g. `us-east1`). Lithops will use the region set under the `gcp` section if it is not set here | |gcp_cloudrun | min_workers | 0 | no | Minimum number of workers of a given runtime to keep in warm status| |gcp_cloudrun | max_workers | 1000 | no | Maximum number of workers to scale a given runtime| diff --git a/docs/source/compute_config/gcp_compute_engine.md b/docs/source/compute_config/gcp_compute_engine.md index 993e463ae..d2c40f516 100644 --- a/docs/source/compute_config/gcp_compute_engine.md +++ b/docs/source/compute_config/gcp_compute_engine.md @@ -68,7 +68,6 @@ gcp: credentials_path: gcp_compute_engine: - project_name: zone: exec_mode: reuse ``` @@ -86,7 +85,7 @@ Lithops attaches the service account from `credentials_path` to master and worke |Group|Key|Default|Mandatory|Additional info| |---|---|---|---|---| -|gcp_compute_engine | project_name | |yes | GCP project ID | +|gcp_compute_engine | project_name | from credentials |no | GCP project ID; overrides `gcp.project_name` | |gcp_compute_engine | zone | |yes | Compute Engine zone, for example `us-east1-b` | |gcp_compute_engine | region | derived from zone |no | Region used for subnet and NAT | |gcp_compute_engine | service_account | |no | Service account email attached to VMs. Default: `client_email` from `credentials_path` | @@ -127,7 +126,6 @@ gcp: gcp_compute_engine: exec_mode: consume - project_name: zone: instance_name: ``` @@ -137,7 +135,7 @@ gcp_compute_engine: |Group|Key|Default|Mandatory|Additional info| |---|---|---|---|---| |gcp_compute_engine | instance_name | |yes | Existing VM instance name | -|gcp_compute_engine | project_name | |yes | GCP project ID | +|gcp_compute_engine | project_name | from credentials |no | GCP project ID; overrides `gcp.project_name` | |gcp_compute_engine | zone | |yes | Compute Engine zone | |gcp_compute_engine | ssh_username | ubuntu |no | Username to access the VM | |gcp_compute_engine | ssh_key_filename | |no | Path to the SSH private key. If not provided, Lithops creates `~/.ssh/lithops-key-.gcp_ce.id_rsa` | diff --git a/docs/source/compute_config/gcp_functions.md b/docs/source/compute_config/gcp_functions.md index ff961d5c4..587908fea 100644 --- a/docs/source/compute_config/gcp_functions.md +++ b/docs/source/compute_config/gcp_functions.md @@ -64,19 +64,21 @@ gcloud services enable \ region : credentials_path : ``` - + ## Summary of configuration keys for Google: ### Google Cloud Platform: |Group|Key|Default|Mandatory|Additional info| |---|---|---|---|---| +|gcp | project_name | from credentials |no | GCP project ID | |gcp | region | |yes | Region name of the GCP services (e.g. `us-east1`) | -|gcp | credentials_path | |yes | **Absolute** path of your JSON key file downloaded in step 7 (e.g. `/home/myuser/lithops-invoker1234567890.json`). Alternatively you can set `GOOGLE_APPLICATION_CREDENTIALS` environment variable. If not provided it will try to load the default credentials from the environment| +|gcp | credentials_path | |no | **Absolute** path of your JSON key file downloaded in step 7 (e.g. `/home/myuser/lithops-invoker1234567890.json`). Alternatively you can set `GOOGLE_APPLICATION_CREDENTIALS` environment variable. If not provided it will try to load the default credentials from the environment| ### Google Cloud Run functions (v2) |Group|Key|Default|Mandatory|Additional info| |---|---|---|---|---| +|gcp_functions | project_name | from gcp or credentials |no | GCP project ID; overrides `gcp.project_name` | |gcp_functions | region | |no | Region name (e.g. `us-east1`). Functions and pub/sub queues will be created in the same region. Lithops will use the region set under the `gcp` section if it is not set here | |gcp_functions | max_workers | 1000 | no | Max number of workers per `FunctionExecutor()`| |gcp_functions | worker_processes | 1 | no | Number of Lithops processes within a given worker. This can be used to parallelize function activations within a worker | diff --git a/lithops/serverless/backends/gcp_cloudrun/cloudrun.py b/lithops/serverless/backends/gcp_cloudrun/cloudrun.py index 1fffeb532..b703b7fb9 100644 --- a/lithops/serverless/backends/gcp_cloudrun/cloudrun.py +++ b/lithops/serverless/backends/gcp_cloudrun/cloudrun.py @@ -116,6 +116,8 @@ def _build_api_resource(self): cred, self.project_name = google.auth.default(scopes=config.SCOPES) self.service_account = cred.service_account_email + self.project_name = self.cr_config.get('project_name') or self.project_name + http = AuthorizedHttp(cred, http=httplib2.Http()) self._api_resource = build( 'run', config.CLOUDRUN_API_VERSION, diff --git a/lithops/serverless/backends/gcp_functions/gcp_functions.py b/lithops/serverless/backends/gcp_functions/gcp_functions.py index d47f48beb..5f6218f0f 100644 --- a/lithops/serverless/backends/gcp_functions/gcp_functions.py +++ b/lithops/serverless/backends/gcp_functions/gcp_functions.py @@ -94,6 +94,8 @@ def _build_api_resource(self): self._pub_client = pubsub_v1.PublisherClient(credentials=pubsub_cred) + self.project_name = self.gcf_config.get('project_name') or self.project_name + http = AuthorizedHttp(api_cred, http=httplib2.Http()) self._api_resource = build( 'cloudfunctions', config.FUNCTIONS_API_VERSION, diff --git a/lithops/standalone/backends/gcp_compute_engine/config.py b/lithops/standalone/backends/gcp_compute_engine/config.py index 4509f70b5..8eca426e6 100644 --- a/lithops/standalone/backends/gcp_compute_engine/config.py +++ b/lithops/standalone/backends/gcp_compute_engine/config.py @@ -13,6 +13,7 @@ # import copy +import json import os import uuid @@ -50,10 +51,26 @@ def load_config(config_data): config_data['gcp_compute_engine'].update(config_data['gcp']) config_data['gcp_compute_engine'].update(temp) - if 'credentials_path' in config_data['gcp_compute_engine']: - config_data['gcp_compute_engine']['credentials_path'] = os.path.expanduser( - config_data['gcp_compute_engine']['credentials_path'] - ) + gce_config = config_data['gcp_compute_engine'] + if not gce_config.get('credentials_path') and os.environ.get('GOOGLE_APPLICATION_CREDENTIALS'): + gce_config['credentials_path'] = os.environ['GOOGLE_APPLICATION_CREDENTIALS'] + if gce_config.get('credentials_path'): + gce_config['credentials_path'] = os.path.expanduser(gce_config['credentials_path']) + + if not gce_config.get('project_name'): + if gce_config.get('credentials_path') and os.path.isfile(gce_config['credentials_path']): + with open(gce_config['credentials_path']) as credentials_file: + project_name = json.load(credentials_file).get('project_id') + else: + import google.auth + from google.auth.exceptions import DefaultCredentialsError + + try: + _, project_name = google.auth.default() + except DefaultCredentialsError: + project_name = None + if project_name: + gce_config['project_name'] = project_name for key in DEFAULT_CONFIG_KEYS: if key not in config_data['gcp_compute_engine']: @@ -75,7 +92,7 @@ def load_config(config_data): params_to_check = MANDATORY_PARAMETERS_2 for param in params_to_check: - if param not in config_data['gcp_compute_engine']: + if not config_data['gcp_compute_engine'].get(param): msg = f"'{param}' is mandatory in the 'gcp_compute_engine' or 'gcp' section of the configuration" raise Exception(msg) diff --git a/lithops/standalone/backends/gcp_compute_engine/gcp_compute_engine.py b/lithops/standalone/backends/gcp_compute_engine/gcp_compute_engine.py index 59c371f41..e95df8402 100644 --- a/lithops/standalone/backends/gcp_compute_engine/gcp_compute_engine.py +++ b/lithops/standalone/backends/gcp_compute_engine/gcp_compute_engine.py @@ -128,7 +128,7 @@ def _resolve_service_account_email(self): 'gcp.credentials_path.' ) - def _wait_operation(self, operation_name, scope='zone'): + def _wait_operation(self, operation_name, scope='zone', ignore_not_found=False): while True: if scope == 'zone': op = self.compute_client.zoneOperations().get( @@ -145,6 +145,9 @@ def _wait_operation(self, operation_name, scope='zone'): if op['status'] == 'DONE': if 'error' in op: + errors = op['error'].get('errors', []) + if ignore_not_found and errors and all(e.get('code') == 'RESOURCE_NOT_FOUND' for e in errors): + return raise Exception(op['error']) return time.sleep(2) @@ -900,10 +903,15 @@ def get_instance_names(): break for name in names: logger.debug(f"Deleting VM instance {name}") - op = self.compute_client.instances().delete( - project=self.project_name, zone=self.zone, instance=name - ).execute() - self._wait_operation(op['name'], scope='zone') + try: + op = self.compute_client.instances().delete( + project=self.project_name, zone=self.zone, instance=name + ).execute() + except HttpError as exc: + if exc.resp.status != 404: + raise + continue + self._wait_operation(op['name'], scope='zone', ignore_not_found=True) def _delete_network_resources(self): """ diff --git a/lithops/tests/test_config.py b/lithops/tests/test_config.py index 36ba4d779..9449c0807 100644 --- a/lithops/tests/test_config.py +++ b/lithops/tests/test_config.py @@ -715,3 +715,64 @@ def test_extract_standalone_does_not_share_standalone_section(self): extracted = extract_standalone_config(cfg) extracted['exec_mode'] = 'consume' assert cfg[c.STANDALONE]['exec_mode'] == 'reuse' + + +class TestGceProjectFallback: + @pytest.mark.parametrize('mode', ['create', 'reuse', 'consume']) + def test_project_from_credentials(self, tmp_path, mode): + from lithops.standalone.backends.gcp_compute_engine.config import load_config + path = tmp_path / 'credentials.json' + path.write_text(json.dumps({'project_id': 'credentials-project'})) + cfg = { + 'gcp': {'credentials_path': str(path)}, + 'gcp_compute_engine': {'zone': 'us-east1-b', 'instance_name': 'vm'}, + 'standalone': {'exec_mode': mode}, + } + load_config(cfg) + assert cfg['gcp_compute_engine']['project_name'] == 'credentials-project' + + @pytest.mark.parametrize('section', ['gcp', 'gcp_compute_engine']) + def test_explicit_project_wins(self, section): + from lithops.standalone.backends.gcp_compute_engine.config import load_config + cfg = {'gcp': {}, 'gcp_compute_engine': {'zone': 'us-east1-b'}} + cfg[section]['project_name'] = 'explicit-project' + with patch('google.auth.default') as adc: + load_config(cfg) + assert cfg['gcp_compute_engine']['project_name'] == 'explicit-project' + adc.assert_not_called() + + def test_backend_project_overrides_shared_project(self): + from lithops.standalone.backends.gcp_compute_engine.config import load_config + cfg = { + 'gcp': {'project_name': 'shared'}, + 'gcp_compute_engine': {'zone': 'us-east1-b', 'project_name': 'backend'}, + } + load_config(cfg) + assert cfg['gcp_compute_engine']['project_name'] == 'backend' + + def test_project_from_adc(self, monkeypatch): + from lithops.standalone.backends.gcp_compute_engine.config import load_config + monkeypatch.delenv('GOOGLE_APPLICATION_CREDENTIALS', raising=False) + cfg = {'gcp_compute_engine': {'zone': 'us-east1-b'}} + with patch('google.auth.default', return_value=(MagicMock(), 'adc-project')): + load_config(cfg) + assert cfg['gcp_compute_engine']['project_name'] == 'adc-project' + + def test_credentials_from_environment_and_home_expansion(self, tmp_path, monkeypatch): + from lithops.standalone.backends.gcp_compute_engine.config import load_config + monkeypatch.setenv('HOME', str(tmp_path)) + monkeypatch.setenv('GOOGLE_APPLICATION_CREDENTIALS', '~/credentials.json') + path = tmp_path / 'credentials.json' + path.write_text(json.dumps({'project_id': 'env-project'})) + cfg = {'gcp_compute_engine': {'zone': 'us-east1-b'}} + load_config(cfg) + assert cfg['gcp_compute_engine']['project_name'] == 'env-project' + assert cfg['gcp_compute_engine']['credentials_path'] == str(path) + + def test_no_resolvable_project_still_raises(self, monkeypatch): + from lithops.standalone.backends.gcp_compute_engine.config import load_config + monkeypatch.delenv('GOOGLE_APPLICATION_CREDENTIALS', raising=False) + cfg = {'gcp_compute_engine': {'zone': 'us-east1-b'}} + with patch('google.auth.default', return_value=(MagicMock(), None)): + with pytest.raises(Exception, match='project_name'): + load_config(cfg) diff --git a/lithops/tests/test_serverless.py b/lithops/tests/test_serverless.py index 7ba4a6c5f..e79cd8d71 100644 --- a/lithops/tests/test_serverless.py +++ b/lithops/tests/test_serverless.py @@ -619,3 +619,33 @@ def item(name): ] assert [row[3] for row in backend.list_runtimes()] == ['a', 'b'] assert services.list.call_args_list[1].kwargs['continue_'] == 'tok' + + +@pytest.mark.parametrize('backend_name', ['gcp_functions', 'gcp_cloudrun']) +@pytest.mark.parametrize('explicit_project', [None, 'explicit-project']) +@pytest.mark.parametrize('use_file', [False, True]) +def test_gcp_project_override_and_credentials_fallback(monkeypatch, backend_name, explicit_project, use_file, tmp_path): + import importlib + + filename, class_name, config_attr = { + 'gcp_functions': ('gcp_functions', 'GCPFunctionsBackend', 'gcf_config'), + 'gcp_cloudrun': ('cloudrun', 'GCPCloudRunBackend', 'cr_config'), + }[backend_name] + module = importlib.import_module(f'lithops.serverless.backends.{backend_name}.{filename}') + backend = getattr(module, class_name).__new__(getattr(module, class_name)) + backend.region = 'us-east1' + credentials_file = tmp_path / 'credentials.json' + credentials_file.write_text('{}') + backend.credentials_path = str(credentials_file) if use_file else None + setattr(backend, config_attr, {'project_name': explicit_project}) + credentials = MagicMock(project_id='credentials-project', service_account_email='service@example.com') + monkeypatch.setattr(module.google.auth, 'default', lambda **kwargs: (credentials, 'adc-project')) + monkeypatch.setattr(module.service_account.Credentials, 'from_service_account_file', lambda *a, **kw: credentials) + monkeypatch.setattr(module, 'AuthorizedHttp', MagicMock()) + monkeypatch.setattr(module, 'build', MagicMock()) + if backend_name == 'gcp_functions': + monkeypatch.setattr(module.pubsub_v1, 'PublisherClient', MagicMock()) + monkeypatch.setattr(module.jwt.Credentials, 'from_service_account_file', MagicMock()) + backend._build_api_resource() + assert backend.project_name == (explicit_project or ('credentials-project' if use_file else 'adc-project')) + assert getattr(backend, config_attr)['project_name'] == backend.project_name diff --git a/lithops/tests/test_standalone.py b/lithops/tests/test_standalone.py index 12ec6f524..2bc00c15f 100644 --- a/lithops/tests/test_standalone.py +++ b/lithops/tests/test_standalone.py @@ -7,6 +7,7 @@ import sys import warnings from contextlib import ExitStack +from types import SimpleNamespace from unittest.mock import MagicMock, patch import pytest @@ -1641,3 +1642,62 @@ def capture(**kwargs): assert ip['publicIPAddress']['id'] == 'ip' else: assert 'publicIPAddress' not in ip + + +class TestGceCleanupMissingInstances: + @staticmethod + def _backend(): + pytest.importorskip('googleapiclient') + from lithops.standalone.backends.gcp_compute_engine.gcp_compute_engine import GCPComputeEngineBackend + backend = GCPComputeEngineBackend.__new__(GCPComputeEngineBackend) + backend.compute_client = MagicMock() + backend.project_name = 'project' + backend.zone = 'us-east1-b' + backend.network_name = 'network' + return backend + + @pytest.mark.parametrize('async_error', [False, True]) + def test_missing_vm_does_not_stop_cleanup(self, async_error): + backend = self._backend() + from googleapiclient.errors import HttpError + backend._iter_zone_instances = MagicMock(side_effect=[[ + {'name': 'lithops-worker-gone'}, {'name': 'lithops-worker-next'}, + ], []]) + delete = backend.compute_client.instances.return_value.delete.return_value.execute + get_operation = backend.compute_client.zoneOperations.return_value.get.return_value.execute + if async_error: + delete.side_effect = [{'name': 'gone-op'}, {'name': 'next-op'}] + get_operation.side_effect = [ + {'status': 'DONE', 'error': {'errors': [{'code': 'RESOURCE_NOT_FOUND'}]}}, + {'status': 'DONE'}, + ] + else: + delete.side_effect = [HttpError(SimpleNamespace(status=404, reason='gone'), b'{}'), {'name': 'next-op'}] + get_operation.return_value = {'status': 'DONE'} + backend._delete_vm_instances() + calls = backend.compute_client.instances.return_value.delete.call_args_list + assert [call.kwargs['instance'] for call in calls] == ['lithops-worker-gone', 'lithops-worker-next'] + + @pytest.mark.parametrize('ignore_not_found,codes', [ + (False, ['RESOURCE_NOT_FOUND']), + (True, ['PERMISSIONS_ERROR']), + (True, ['RESOURCE_NOT_FOUND', 'PERMISSIONS_ERROR']), + (True, []), + ]) + def test_other_operation_errors_still_raise(self, ignore_not_found, codes): + backend = self._backend() + backend.compute_client.zoneOperations.return_value.get.return_value.execute.return_value = { + 'status': 'DONE', 'error': {'errors': [{'code': code} for code in codes]}, + } + with pytest.raises(Exception): + backend._wait_operation('op', ignore_not_found=ignore_not_found) + + def test_permission_failure_still_raises(self): + backend = self._backend() + from googleapiclient.errors import HttpError + backend._iter_zone_instances = MagicMock(return_value=[{'name': 'lithops-worker-denied'}]) + backend.compute_client.instances.return_value.delete.return_value.execute.side_effect = HttpError( + SimpleNamespace(status=403, reason='denied'), b'{}', + ) + with pytest.raises(HttpError): + backend._delete_vm_instances() From fac5ae8e4e35e37c63a6fc60f6aba1cf415830b4 Mon Sep 17 00:00:00 2001 From: JosepSampe Date: Tue, 27 Oct 2026 18:01:30 +0100 Subject: [PATCH 12/13] Fix serialize --- CHANGELOG.md | 3 + docs/source/compute_config/code_engine.md | 2 +- lithops/job/job.py | 4 +- lithops/job/serialize.py | 14 ++-- .../backends/code_engine/code_engine.py | 49 +++++++++++- lithops/tests/test_ibm_openwhisk_backends.py | 77 +++++++++++++++++++ lithops/tests/test_job.py | 4 +- lithops/tests/test_worker.py | 20 ++--- lithops/worker/handler.py | 19 ++--- 9 files changed, 156 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57eeace72..2890f54ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ ## [v3.8.1.dev0] ### Changed +- [Worker] Start status is sent before launching the JobRunner process. +- [Core] Dependency debug logs distinguish function modules from input-data modules. - [Azure Functions] Default runtime memory is now 512 MB. - [Core] A storage or monitoring service at a loopback address is refused for serverless workers, which cannot reach it; standalone and Singularity workers get a warning. - [Core] `wait()` timeouts no longer arm a SIGALRM: they are checked between polls, work from any thread and leave the caller's signal handler and alarm alone. @@ -13,6 +15,7 @@ - [Knative] Updated to current Knative Serving (tested with 1.23): one container per call in flight with the current autoscaling annotations, `kubecfg_context`, `namespace` and `runtime_arch` keys, the Contour networking layer, cluster-local calls from inside the cluster, a deploy that updates an existing service and reports why its revision fails, and a default runtime that runs as a non-root user. ### Fixed +- [Code Engine] `clean --all` permanently deletes the project so its name can be reused. - [GCP Compute Engine] Cleanup tolerates VMs already deleted by another process. - [GCP] Compute Engine infers the project from credentials when omitted; Functions and Cloud Run honour explicit project IDs. - [GCP Functions] Cleaning runtimes works when service configuration is missing. diff --git a/docs/source/compute_config/code_engine.md b/docs/source/compute_config/code_engine.md index 717316bd9..52566bffc 100644 --- a/docs/source/compute_config/code_engine.md +++ b/docs/source/compute_config/code_engine.md @@ -128,7 +128,7 @@ Delete Lithops runtimes, job runs, and cached metadata: lithops clean -b code_engine -s ibm_cos ``` -To also delete the Code Engine project and local cache: +To permanently delete the Code Engine project and local cache: ```bash lithops clean -b code_engine -s ibm_cos --all diff --git a/lithops/job/job.py b/lithops/job/job.py index f6b0a422a..557355b2b 100644 --- a/lithops/job/job.py +++ b/lithops/job/job.py @@ -73,12 +73,12 @@ def _serialize_func_and_data( upload time; callable identity alone cannot detect changed captured state """ func_str = serializer.dumps([func])[0] - func_paths = serializer.module_paths([func], inc_modules, exc_modules) + func_paths = serializer.module_paths([func], inc_modules, exc_modules, source='Function') data_strs = serializer.dumps(iterdata) # The data is only inspected for modules when the module manager is on # and no explicit include list was given if inc_modules is not None and not inc_modules: - data_paths = serializer.module_paths(iterdata, inc_modules, exc_modules) + data_paths = serializer.module_paths(iterdata, inc_modules, exc_modules, source='Input data') else: data_paths = set() return func_str, data_strs, func_paths | data_paths diff --git a/lithops/job/serialize.py b/lithops/job/serialize.py index 3a729c589..1fa1e21be 100644 --- a/lithops/job/serialize.py +++ b/lithops/job/serialize.py @@ -94,7 +94,8 @@ def _preinstalled_names(self) -> Set[str]: def _referenced_module_paths( self, list_of_objs: List, - exclude_modules: Iterable[str] + exclude_modules: Iterable[str], + source: str = 'Objects' ) -> Set[str]: """ Finds the paths of the modules the objects reference, leaving out the @@ -108,7 +109,7 @@ def _referenced_module_paths( for obj in list_of_objs: ref_modules.update(self._module_inspect(obj)) - logger.debug(f"Referenced Modules: {_joined_or_none(ref_modules)}") + logger.debug(f"{source} - Referenced Modules: {_joined_or_none(ref_modules)}") mod_paths = set() for module_name in ref_modules: @@ -179,24 +180,25 @@ def module_paths( self, list_of_objs: List, include_modules: Optional[Iterable[str]], - exclude_modules: Iterable[str] + exclude_modules: Iterable[str], + source: str = 'Objects' ) -> Set[str]: """ Collects the paths of the modules that have to travel with the job: either the ones explicitly included, or the ones its code references """ if include_modules is None: - logger.debug('Module manager disabled. Modules to transmit: None') + logger.debug(f'{source} - Module manager disabled. Modules to transmit: None') return set() if include_modules: mod_paths = self._explicit_module_paths(include_modules) else: mod_paths = self._referenced_module_paths( - list_of_objs, exclude_modules + list_of_objs, exclude_modules, source ) - logger.debug(f"Modules to transmit: {_joined_or_none(mod_paths)}") + logger.debug(f"{source} - Modules to transmit: {_joined_or_none(mod_paths)}") return mod_paths diff --git a/lithops/serverless/backends/code_engine/code_engine.py b/lithops/serverless/backends/code_engine/code_engine.py index 0349140b7..356d76bf5 100644 --- a/lithops/serverless/backends/code_engine/code_engine.py +++ b/lithops/serverless/backends/code_engine/code_engine.py @@ -23,7 +23,7 @@ import logging import copy -from ibm_cloud_sdk_core import ApiException +from ibm_cloud_sdk_core import ApiException, BaseService from ibm_code_engine_sdk.code_engine_v2 import ( CodeEngineV2, EnvVarPrototype, @@ -773,12 +773,57 @@ def delete_runtime(self, runtime_name, memory, version=__version__, jobdef_name= logger.info(f'Deleting runtime: {runtime_name} - {memory}MB') self._delete_job_definition(jobdef_name) + def _hard_delete_project(self, project_id): + """Permanently reclaim the project before removing its local cache.""" + try: + self.ce_client.delete_project(id=project_id) + except ApiException as exc: + if exc.status_code != 404: + raise + + controller = BaseService( + service_url='https://resource-controller.cloud.ibm.com', + authenticator=self.ce_client.get_authenticator(), + ) + deadline = time.time() + config.PROJECT_READY_TIMEOUT + reclaimed = False + while time.time() < deadline: + request = controller.prepare_request('GET', f'/v2/resource_instances/{project_id}') + try: + instance = controller.send(request).get_result() + except ApiException as exc: + if exc.status_code == 404: + return + raise + if instance.get('state') == 'removed': + return + if not reclaimed: + request = controller.prepare_request( + 'GET', '/v1/reclamations', params={'resource_instance_id': project_id} + ) + reclamations = controller.send(request).get_result().get('resources', []) + for reclamation in reclamations: + if reclamation.get('resource_instance_id') != project_id: + continue + request = controller.prepare_request( + 'POST', f'/v1/reclamations/{reclamation["id"]}/actions/reclaim', + headers={'Content-Type': 'application/json'}, data={}, + ) + controller.send(request) + reclaimed = True + break + time.sleep(config.PROJECT_POLL_INTERVAL) + raise TimeoutError(f'Timed out permanently deleting Code Engine project {self.project_name}') + def clean(self, all=False): """ Deletes all runtimes from all packages """ logger.info(f'Cleaning project {self.project_name}') + cached_project_id = self.project_id or load_yaml_config(self.cache_file).get('project_id') if not self._get_or_create_namespace(create=False, wait_for_active=False): + if all and cached_project_id: + self._hard_delete_project(cached_project_id) logger.info(f"Project {self.project_name} does not exist") if os.path.exists(self.cache_file): os.remove(self.cache_file) @@ -792,7 +837,7 @@ def clean(self, all=False): if all and self.project_id: logger.info(f"Deleting Code Engine project: {self.project_name}") - self.ce_client.delete_project(id=self.project_id) + self._hard_delete_project(self.project_id) if os.path.exists(self.cache_file): os.remove(self.cache_file) diff --git a/lithops/tests/test_ibm_openwhisk_backends.py b/lithops/tests/test_ibm_openwhisk_backends.py index f2f4b199a..0e6f038ed 100644 --- a/lithops/tests/test_ibm_openwhisk_backends.py +++ b/lithops/tests/test_ibm_openwhisk_backends.py @@ -225,3 +225,80 @@ def test_singularity_runtime_meta_timeout_message(monkeypatch): with pytest.raises(Exception, match='Failed getting runtime metadata'): backend._generate_runtime_meta('singularity-runtime') + + +@pytest.mark.parametrize('already_deleted', [False, True]) +def test_code_engine_project_is_permanently_reclaimed(ce, monkeypatch, already_deleted): + if already_deleted: + ce.backend.ce_client.delete_project.side_effect = ce.ApiException(404, message='gone') + controller = MagicMock() + monkeypatch.setattr(ce.module, 'BaseService', MagicMock(return_value=controller)) + monkeypatch.setattr(ce.module.time, 'sleep', lambda seconds: None) + controller.prepare_request.side_effect = lambda method, url, **kwargs: {'method': method, 'url': url, **kwargs} + controller.send.side_effect = [ + MagicMock(**{'get_result.return_value': {'state': 'pending_reclamation'}}), + MagicMock(**{'get_result.return_value': {'resources': []}}), + MagicMock(**{'get_result.return_value': {'state': 'pending_reclamation'}}), + MagicMock(**{'get_result.return_value': {'resources': [ + {'id': 'unrelated', 'resource_instance_id': 'other-project'}, + {'id': 'reclamation-id', 'resource_instance_id': 'project-id'}, + ]}}), + MagicMock(), + MagicMock(**{'get_result.return_value': {'state': 'removed'}}), + ] + ce.backend._hard_delete_project('project-id') + ce.backend.ce_client.delete_project.assert_called_once_with(id='project-id') + requests = [call.args[0] for call in controller.send.call_args_list] + posts = [request for request in requests if request['method'] == 'POST'] + assert posts == [{ + 'method': 'POST', 'url': '/v1/reclamations/reclamation-id/actions/reclaim', + 'headers': {'Content-Type': 'application/json'}, 'data': {}, + }] + assert requests[1]['params'] == {'resource_instance_id': 'project-id'} + + +@pytest.mark.parametrize('all_clean', [False, True]) +@pytest.mark.parametrize('active', [False, True]) +def test_code_engine_clean_hard_deletes_only_with_all(ce, monkeypatch, tmp_path, all_clean, active): + backend = ce.backend + backend.cache_file = str(tmp_path / 'cache') + ce.module.dump_yaml_config(backend.cache_file, {'project_id': 'project-id'}) + + def namespace(**kwargs): + if not active: + backend.project_id = None + return 'namespace' if active else None + + monkeypatch.setattr(backend, '_get_or_create_namespace', namespace) + monkeypatch.setattr(backend, 'clear', MagicMock()) + monkeypatch.setattr(backend, 'list_runtimes', lambda: []) + monkeypatch.setattr(backend, '_delete_lithops_config_maps', MagicMock()) + hard_delete = MagicMock() + monkeypatch.setattr(backend, '_hard_delete_project', hard_delete) + backend.clean(all=all_clean) + if all_clean: + hard_delete.assert_called_once_with('project-id') + else: + hard_delete.assert_not_called() + + +@pytest.mark.parametrize('failure', ['permission', 'timeout']) +def test_code_engine_failed_hard_delete_keeps_cache(ce, monkeypatch, tmp_path, failure): + backend = ce.backend + backend.cache_file = str(tmp_path / 'cache') + ce.module.dump_yaml_config(backend.cache_file, {'project_id': 'project-id'}) + monkeypatch.setattr(backend, '_get_or_create_namespace', lambda **kwargs: 'namespace') + monkeypatch.setattr(backend, 'clear', MagicMock()) + monkeypatch.setattr(backend, 'list_runtimes', lambda: []) + monkeypatch.setattr(backend, '_delete_lithops_config_maps', MagicMock()) + controller = MagicMock() + monkeypatch.setattr(ce.module, 'BaseService', MagicMock(return_value=controller)) + if failure == 'permission': + controller.send.side_effect = ce.ApiException(403, message='forbidden') + expected = ce.ApiException + else: + monkeypatch.setattr(ce.module.config, 'PROJECT_READY_TIMEOUT', 0) + expected = TimeoutError + with pytest.raises(expected): + backend.clean(all=True) + assert (tmp_path / 'cache').exists() diff --git a/lithops/tests/test_job.py b/lithops/tests/test_job.py index 1e5bc7111..3d7aea26d 100644 --- a/lithops/tests/test_job.py +++ b/lithops/tests/test_job.py @@ -91,7 +91,7 @@ def dumps(self, objs): self.objs.extend(objs) return strs - def module_paths(self, objs, inc_modules, exc_modules): + def module_paths(self, objs, inc_modules, exc_modules, source='Objects'): type(self).last = (inc_modules, exc_modules, self.objs) return set() @@ -474,7 +474,7 @@ def dumps(self, objs): return [b'f'] return [b'x' * (MAX_DATA_IN_PAYLOAD + 1)] * len(objs) - def module_paths(self, objs, inc, exc): + def module_paths(self, objs, inc, exc, source='Objects'): return set() monkeypatch.setattr('lithops.job.job.SerializeIndependent', _Big) diff --git a/lithops/tests/test_worker.py b/lithops/tests/test_worker.py index 9fa9ec6a5..9ff328e34 100644 --- a/lithops/tests/test_worker.py +++ b/lithops/tests/test_worker.py @@ -549,9 +549,15 @@ def test_success_reads_stats_and_sends_events(self, tmp_path): jrp.is_alive.return_value = False conn = MagicMock() conn.poll.return_value = True - status = self._patch_run( - task, jrp, conn, 'worker_func_exec_time 1.5\nexception True\n' + events = [] + status = MagicMock() + status.send_init_event.side_effect = lambda: events.append('init') + jrp.start.side_effect = lambda: events.append('start') + status.send_finish_event.side_effect = lambda: events.append('finish') + self._patch_run( + task, jrp, conn, 'worker_func_exec_time 1.5\nexception True\n', status=status ) + assert events == ['init', 'start', 'finish'] status.send_init_event.assert_called_once() status.send_finish_event.assert_called_once() added = {c.args[0]: c.args[1] for c in status.add.call_args_list} @@ -634,10 +640,8 @@ def test_a_failed_start_report_still_waits_for_the_function( self, tmp_path ): """ - The init event is sent once the JobRunner runs. When sending it - fails, say the status object cannot be written to storage, the call - used to be reported as failed while the JobRunner kept running, - never joined, next to the following task of the worker + An init reporting failure must not prevent function execution or + reporting its result. """ task = _task() task.log_stream = MagicMock() @@ -704,9 +708,6 @@ def test_keyboard_interrupt_skips_finish_event(self, tmp_path): task.stats_file = str(tmp_path / 'missing.txt') status = MagicMock() status.send_init_event.side_effect = KeyboardInterrupt() - # The init event is sent after the JobRunner has been started, so - # the context is mocked out like the other tests here do: without - # it this unit test forks a real process to interrupt jrp = MagicMock() jrp.is_alive.return_value = False ctx = MagicMock() @@ -726,6 +727,7 @@ def test_keyboard_interrupt_skips_finish_event(self, tmp_path): run_task(task) status.send_init_event.assert_called_once() status.send_finish_event.assert_not_called() + jrp.start.assert_not_called() def test_runtime_memory_log_branch(self, tmp_path): task = _task(runtime_memory=256) diff --git a/lithops/worker/handler.py b/lithops/worker/handler.py index afd30a38f..8bb88e21d 100644 --- a/lithops/worker/handler.py +++ b/lithops/worker/handler.py @@ -516,6 +516,11 @@ def run_task(task: SimpleNamespace) -> None: try: handler_conn, jobrunner_conn = _MP_CTX.Pipe() jobrunner = JobRunner(task, jobrunner_conn, internal_storage) + try: + call_status.send_init_event() + except Exception as e: + logger.warning(f'Could not report the start of the call: {e}') + logger.debug('Starting JobRunner process') jrp = ( _MP_CTX.Process(target=jobrunner.run) @@ -527,20 +532,6 @@ def run_task(task: SimpleNamespace) -> None: sys_monitor.start() jrp.start() - - # Reported once the process is running, which is also when the call - # has really started. Sending it here rather than before the fork - # keeps the one-off cost of opening the monitoring client off the - # critical path: the first call of a worker pays for a connection - # (some 13 ms for AMQP, then kept for the whole process) while the - # function is already running, instead of delaying its start. - # The event is informational and the finish event reports the call - # all the same, so failing to send it must not abandon a JobRunner - # that is already running - try: - call_status.send_init_event() - except Exception as e: - logger.warning(f'Could not report the start of the call: {e}') jrp.join(task.execution_timeout) sys_monitor.stop() From b94e53deac4545bf6444d34e7aa50c21a56dbead Mon Sep 17 00:00:00 2001 From: JosepSampe Date: Tue, 27 Oct 2026 19:55:18 +0100 Subject: [PATCH 13/13] Fix tests --- lithops/tests/test_azure_aliyun_oracle_backends.py | 4 ++-- lithops/tests/test_config.py | 5 +++++ lithops/tests/test_serverless.py | 10 ++++++++++ lithops/tests/test_standalone.py | 7 +++++++ lithops/tests/test_storage_layer.py | 3 +-- 5 files changed, 25 insertions(+), 4 deletions(-) diff --git a/lithops/tests/test_azure_aliyun_oracle_backends.py b/lithops/tests/test_azure_aliyun_oracle_backends.py index e2f8d90ce..c0aee1590 100644 --- a/lithops/tests/test_azure_aliyun_oracle_backends.py +++ b/lithops/tests/test_azure_aliyun_oracle_backends.py @@ -296,8 +296,8 @@ def run_az(cmd, return_json=False, **kw): monkeypatch.setattr(backend, '_run_az_command', run_az) backend.queue_service = types.SimpleNamespace(delete_queue=lambda name: None) - assert backend.list_runtimes('rt') == [('rt', 2048, __version__, legacy_name)] - backend.delete_runtime('rt', 2048, __version__) + assert backend.list_runtimes('rt') == [('rt', 512, __version__, legacy_name)] + backend.delete_runtime('rt', 512, __version__) assert commands == [f'az functionapp delete --name {legacy_name} --resource-group rg'] diff --git a/lithops/tests/test_config.py b/lithops/tests/test_config.py index 9449c0807..19fe15c24 100644 --- a/lithops/tests/test_config.py +++ b/lithops/tests/test_config.py @@ -718,6 +718,11 @@ def test_extract_standalone_does_not_share_standalone_section(self): class TestGceProjectFallback: + @pytest.fixture(autouse=True) + def _need_gcp(self): + pytest.importorskip('googleapiclient') + pytest.importorskip('google_auth_httplib2') + @pytest.mark.parametrize('mode', ['create', 'reuse', 'consume']) def test_project_from_credentials(self, tmp_path, mode): from lithops.standalone.backends.gcp_compute_engine.config import load_config diff --git a/lithops/tests/test_serverless.py b/lithops/tests/test_serverless.py index e79cd8d71..d2a73aa27 100644 --- a/lithops/tests/test_serverless.py +++ b/lithops/tests/test_serverless.py @@ -389,6 +389,12 @@ def http_error(status): class TestGcpFunctionsDeploy: + @pytest.fixture(autouse=True) + def _need_gcp(self): + pytest.importorskip('googleapiclient') + pytest.importorskip('google_auth_httplib2') + pytest.importorskip('google.cloud.pubsub_v1') + def _backend(self): from lithops.serverless.backends.gcp_functions.gcp_functions import GCPFunctionsBackend backend = GCPFunctionsBackend.__new__(GCPFunctionsBackend) @@ -631,6 +637,10 @@ def test_gcp_project_override_and_credentials_fallback(monkeypatch, backend_name 'gcp_functions': ('gcp_functions', 'GCPFunctionsBackend', 'gcf_config'), 'gcp_cloudrun': ('cloudrun', 'GCPCloudRunBackend', 'cr_config'), }[backend_name] + pytest.importorskip('googleapiclient') + pytest.importorskip('google_auth_httplib2') + if backend_name == 'gcp_functions': + pytest.importorskip('google.cloud.pubsub_v1') module = importlib.import_module(f'lithops.serverless.backends.{backend_name}.{filename}') backend = getattr(module, class_name).__new__(getattr(module, class_name)) backend.region = 'us-east1' diff --git a/lithops/tests/test_standalone.py b/lithops/tests/test_standalone.py index 2bc00c15f..cddffc2d8 100644 --- a/lithops/tests/test_standalone.py +++ b/lithops/tests/test_standalone.py @@ -736,6 +736,7 @@ def test_ibm_deletes_a_floating_ip_it_allocated(self): backend.vpc_cli.delete_floating_ip.assert_called_once_with(id='fip-new') def test_aws_spot_price_without_history_raises(self): + pytest.importorskip('boto3') from lithops.standalone.backends.aws_ec2.aws_ec2 import AWSEC2Backend backend = AWSEC2Backend.__new__(AWSEC2Backend) backend.config = { @@ -750,6 +751,7 @@ def test_aws_spot_price_without_history_raises(self): backend._request_spot_price() def test_delete_vms_reads_every_page_of_instances(self): + pytest.importorskip('boto3') from lithops.standalone.backends.aws_ec2.aws_ec2 import AWSEC2Backend backend = AWSEC2Backend.__new__(AWSEC2Backend) backend.vpc_name = 'lithops-vpc' @@ -779,6 +781,8 @@ def instance(instance_id, name): ) def test_gcp_clean_reads_every_page_of_instances(self): + pytest.importorskip('googleapiclient') + pytest.importorskip('google_auth_httplib2') from lithops.standalone.backends.gcp_compute_engine.gcp_compute_engine import GCPComputeEngineBackend backend = GCPComputeEngineBackend.__new__(GCPComputeEngineBackend) backend.network_name = 'lithops-net' @@ -822,6 +826,7 @@ def _vpc_page(items, key, start=None): return response def test_ibm_clean_reads_every_page_of_instances(self): + pytest.importorskip('ibm_vpc') from lithops.standalone.backends.ibm_vpc.ibm_vpc import IBMVPCBackend backend = IBMVPCBackend.__new__(IBMVPCBackend) backend.vpc_name = 'lithops-vpc' @@ -849,6 +854,7 @@ def test_ibm_clean_reads_every_page_of_instances(self): assert deleted == {'i-1', 'i-2'} def test_ibm_image_lookup_reads_the_page_after_the_first(self): + pytest.importorskip('ibm_vpc') from lithops.standalone.backends.ibm_vpc.ibm_vpc import IBMVPCBackend backend = IBMVPCBackend.__new__(IBMVPCBackend) backend.config = {'resource_group_id': 'rg'} @@ -868,6 +874,7 @@ def test_ibm_image_lookup_reads_the_page_after_the_first(self): assert backend.vpc_cli.list_images.call_args_list[1].kwargs['start'] == 'tok' def test_ibm_reuses_a_floating_ip_past_the_first_page(self): + pytest.importorskip('ibm_vpc') from lithops.standalone.backends.ibm_vpc.ibm_vpc import IBMVPCBackend backend = IBMVPCBackend.__new__(IBMVPCBackend) backend.config = {'zone_name': 'eu-de-1', 'resource_group_id': 'rg'} diff --git a/lithops/tests/test_storage_layer.py b/lithops/tests/test_storage_layer.py index cc7271ede..54749799f 100644 --- a/lithops/tests/test_storage_layer.py +++ b/lithops/tests/test_storage_layer.py @@ -662,8 +662,7 @@ def page(names, next_start): assert backend.os_client.list_objects.call_args.kwargs['start'] == 'c' def test_oracle_list_keeps_non_404_errors(self): - import oci - pytest.importorskip('oci') + oci = pytest.importorskip('oci') from lithops.storage.backends.oracle_oss import StorageBackend backend = StorageBackend.__new__(StorageBackend) backend.namespace = 'ns'