Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
113 changes: 106 additions & 7 deletions lib/cloud_controller/blobstore/base_client.rb
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,39 @@ module CloudController
module Blobstore
class BaseClient
def cp_r_to_blobstore(source_dir)
Find.find(source_dir).each do |path|
next unless File.file?(path)
next unless within_limits?(File.size(path))
next unless File.stat(path).mode.to_s(8)[3..5].to_i(8) >= 0o600
counters = new_cp_r_counters
traversal_completed = false

sha1 = Digester.new.digest_path(path)
next if exists?(sha1)
begin
unless File.directory?(source_dir)
logger.error('blobstore.cp_r.source_dir_missing', source_dir: source_dir)
raise Errno::ENOENT.new(source_dir)
end

cp_to_blobstore(path, sha1)
Find.find(source_dir).each do |path|
begin
stat = File.stat(path)
rescue SystemCallError
next
end
next unless stat.file?
next unless within_limits?(stat.size)
next unless stat.mode.to_s(8)[3..5].to_i(8) >= 0o600

cp_r_digest_file(path, counters)
sha1 = counters[:last_sha1]

next unless sha1

exists = cp_r_check_exists(sha1, counters)
next if exists.nil? # exists? raised, already logged
next if exists

cp_r_upload_file(path, sha1, counters)
end
traversal_completed = true
ensure
log_cp_r_phases(counters, traversal_completed)
end
end

Expand All @@ -23,6 +47,77 @@ def cp_to_blobstore(_, _)

private

def new_cp_r_counters
{ digest_duration: 0, exists_duration: 0, upload_duration: 0,
digest_count: 0, exists_count: 0, upload_count: 0,
upload_attempts: 0, exists_errors: 0, last_sha1: nil }
end

def cp_r_digest_file(path, counters)
t = Time.now.utc
counters[:digest_count] += 1
counters[:last_sha1] = Digester.new.digest_path(path)
ensure
counters[:digest_duration] += Time.now.utc - t
end

def cp_r_check_exists(sha1, counters)
t = Time.now.utc
counters[:exists_count] += 1
exists?(sha1)
rescue StandardError => e
counters[:exists_errors] += 1
logger.error('blobstore.cp_r.exists.error', sha1: sha1, error: e.message)
nil
ensure
counters[:exists_duration] += Time.now.utc - t
end

def cp_r_upload_file(path, sha1, counters)
t = Time.now.utc
counters[:upload_attempts] += 1
cp_to_blobstore(path, sha1)
counters[:upload_count] += 1
ensure
counters[:upload_duration] += Time.now.utc - t
end

def log_cp_r_phases(ctrs, completed)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Consider refactoring this function, the math calculations are mixed into the logging, which makes it a little hard to follow. Here is example refactored code might look like:

def log_cp_r_phases(ctrs, completed)
        logger.info('blobstore.cp_r.digest',
                    files_scanned: ctrs[:digest_count],
                    duration_seconds: ctrs[:digest_duration],
                    completed: completed)

        log_exists_phase(ctrs, completed)

        ua = ctrs[:upload_attempts]
        logger.info('blobstore.cp_r.upload',
                    files_to_upload: ua,
                    files_uploaded: ctrs[:upload_count],
                    files_failed: ua - ctrs[:upload_count],
                    duration_seconds: ctrs[:upload_duration],
                    completed: completed)
      end

def log_exists_phase(ctrs, completed)
        checks   = ctrs[:exists_count]
        attempts = ctrs[:upload_attempts]
        errors   = ctrs[:exists_errors]
        hits     = checks - attempts - errors
        resolved = checks - errors # checks that did not error

        logger.info('blobstore.cp_r.exists',
                    files_checked: checks,
                    duration_seconds: ctrs[:exists_duration],
                    avg_duration_seconds: safe_div(ctrs[:exists_duration], checks),
                    cache_hits: hits,
                    cache_misses: attempts,
                    cache_hit_rate: safe_div(hits, resolved),
                    process_spawns: checks,
                    error_count: errors,
                    completed: completed)
      end

      def safe_div(num, denom) = denom.positive? ? num.to_f / denom : 0

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Worth doing. I will do the refactor.

logger.info('blobstore.cp_r.digest',
files_scanned: ctrs[:digest_count],
duration_seconds: ctrs[:digest_duration],
completed: completed)
log_exists_phase(ctrs, completed)
ua = ctrs[:upload_attempts]
logger.info('blobstore.cp_r.upload',
files_to_upload: ua,
files_uploaded: ctrs[:upload_count],
files_failed: ua - ctrs[:upload_count],
duration_seconds: ctrs[:upload_duration],
completed: completed)
end

def log_exists_phase(ctrs, completed)
checks = ctrs[:exists_count]
attempts = ctrs[:upload_attempts]
errors = ctrs[:exists_errors]
hits = checks - attempts - errors
resolved = checks - errors # checks that did not error

logger.info('blobstore.cp_r.exists',
files_checked: checks,
duration_seconds: ctrs[:exists_duration],
avg_duration_seconds: safe_div(ctrs[:exists_duration], checks),
cache_hits: hits,
cache_misses: attempts,
cache_hit_rate: safe_div(hits, resolved),
process_spawns: checks,
error_count: errors,
completed: completed)
end

def safe_div(num, denom) = denom.positive? ? num.to_f / denom : 0

def partitioned_key(key)
key = key.to_s.downcase
partitioned_key = BlobKeyGenerator.full_path_from_key(key)
Expand All @@ -33,6 +128,10 @@ def partitioned_key(key)
def within_limits?(size)
size >= @min_size && (@max_size.nil? || size <= @max_size)
end

def logger
@logger ||= Steno.logger('cc.blobstore')
end
end
end
end
12 changes: 11 additions & 1 deletion lib/cloud_controller/blobstore/client_provider.rb
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
require 'cloud_controller/blobstore/local/local_client'
require 'cloud_controller/blobstore/safe_delete_client'
require 'cloud_controller/blobstore/storage_cli/storage_cli_client'
require 'cloud_controller/execution_context'
require 'cloud_controller/metrics/prometheus_updater'

module CloudController
module Blobstore
Expand Down Expand Up @@ -49,7 +51,8 @@ def provide_storage_cli(options, directory_key, root_dir, resource_type)
resource_type: resource_type,
root_dir: root_dir,
min_size: options[:minimum_size],
max_size: options[:maximum_size]
max_size: options[:maximum_size],
metrics_updater: metrics_updater_if_registered
)

logger = Steno.logger('cc.blobstore.storage_cli_client')
Expand All @@ -58,6 +61,13 @@ def provide_storage_cli(options, directory_key, root_dir, resource_type)

Client.new(SafeDeleteClient.new(retryable_client, root_dir))
end

def metrics_updater_if_registered
ctx = VCAP::CloudController::ExecutionContext.from_process_type_env
return nil unless VCAP::CloudController::Metrics::PrometheusUpdater::STORAGE_CLI_METRICS_EXECUTION_CONTEXTS.include?(ctx)

CloudController::DependencyLocator.instance.prometheus_updater
end
end
end
end
Expand Down
Loading
Loading