diff --git a/README.md b/README.md index f219f30..822ea77 100644 --- a/README.md +++ b/README.md @@ -197,6 +197,17 @@ Crawl delays can be configured (in seconds) either globally for all sites or on Solr's commitWithin option can be configured (in milliseconds) by passing a value under the commit_within key. +#### Harvesting Documents Without Indexing Them + +The harvester exposes the documents it harvests as an enumerable, so you can do something with them other than index them into Solr -- writing them to disk to contribute them to your own OpenGeoMetadata repository, for example. The documents yielded have already been through the configured document transformer. + +```ruby +harvester = GeoCombine::GeoBlacklightHarvester.new(:SITE1) +harvester.each_document do |document| + File.write("#{document['id']}.json", JSON.pretty_generate(document)) +end +``` + #### Transforming Documents You may need to transform documents that are harvested for various purposes (removing fields, adding fields, omitting a document all together, etc). You can configure some ruby code (a proc) that will take the document in, transform it, and return the transformed document. By default the indexer will remove the `score`, `timestamp`, and `_version_` fields from the documents harvested. If you provide your own transformer, you'll likely want to remove these fields in addition to the other transformations you provide. diff --git a/lib/geo_combine/geo_blacklight_harvester.rb b/lib/geo_combine/geo_blacklight_harvester.rb index 7fe9e0d..72d1b19 100644 --- a/lib/geo_combine/geo_blacklight_harvester.rb +++ b/lib/geo_combine/geo_blacklight_harvester.rb @@ -55,25 +55,25 @@ def initialize(site_key, logger: GeoCombine::Logger.logger) raise ArgumentError, "Site key #{@site_key.inspect} is not configured for #{self.class.name}" unless @site end + # Index the documents harvested from the site into Solr def index - @logger.debug "fetching page 1 @ #{base_url}&page=1" - response = JSON.parse(Net::HTTP.get(URI("#{base_url}&page=1"))) - response_class = BlacklightResponseVersionFactory.call(response) - - response_class.new(response:, base_url:, logger: @logger).documents.each do |docs| - docs.map! do |document| - self.class.document_transformer&.call(document) - end.compact - - @logger.debug "adding #{docs.count} documents to solr" + each_page do |documents| + @logger.debug "adding #{documents.count} documents to solr" solr_connection.update params: { commitWithin: commit_within, overwrite: true }, - data: docs.to_json, + data: documents.to_json, headers: { 'Content-Type' => 'application/json' } - - sleep(crawl_delay.to_i) if crawl_delay end end + # Enumerable of the documents harvested from the site, for passing to an + # indexer or doing something else with them (e.g. writing them to disk). + # Documents have already been through the configured document transformer. + def each_document(&block) + return to_enum(:each_document) unless block_given? + + each_page { |documents| documents.each(&block) } + end + ## # A "factory" class to determine the blacklight response version to use class BlacklightResponseVersionFactory @@ -188,6 +188,21 @@ def documents_from_urls(urls) private + # Enumerable of pages of transformed documents harvested from the site + def each_page + return to_enum(:each_page) unless block_given? + + @logger.debug "fetching page 1 @ #{base_url}&page=1" + response = JSON.parse(Net::HTTP.get(URI("#{base_url}&page=1"))) + response_class = BlacklightResponseVersionFactory.call(response) + + response_class.new(response:, base_url:, logger: @logger).documents.each do |documents| + yield documents.map { |document| self.class.document_transformer&.call(document) }.compact + + sleep(crawl_delay.to_i) if crawl_delay + end + end + def base_url "#{site[:host]}?#{default_params.to_query}" end diff --git a/spec/lib/geo_combine/geo_blacklight_harvester_spec.rb b/spec/lib/geo_combine/geo_blacklight_harvester_spec.rb index 6b0b0d3..23457a8 100644 --- a/spec/lib/geo_combine/geo_blacklight_harvester_spec.rb +++ b/spec/lib/geo_combine/geo_blacklight_harvester_spec.rb @@ -120,6 +120,40 @@ end end + describe '#each_document' do + before do + expect(Net::HTTP).to receive(:get).with( + URI('https://example.com?f%5Bdct_provenance_s%5D%5B%5D=INSTITUTION&format=json&per_page=100&page=1') + ).and_return(stub_json_response) + end + + let(:docs) { [{ 'layer_slug_s' => 'abc-123', 'score' => 0.1 }, { 'layer_slug_s' => 'abc-321' }] } + let(:transformed_docs) { [{ 'layer_slug_s' => 'abc-123' }, { 'layer_slug_s' => 'abc-321' }] } + let(:stub_json_response) do + { response: { docs:, pages: { current_page: 1, total_pages: 1 } } }.to_json + end + + it 'yields each transformed document' do + expect { |block| harvester.each_document(&block) }.to yield_successive_args(*transformed_docs) + end + + it 'returns an enumerator when no block is given' do + expect(harvester.each_document.to_a).to eq(transformed_docs) + end + + context 'when the document transformer omits a document' do + before do + allow(described_class).to receive(:document_transformer).and_return( + ->(document) { document unless document['layer_slug_s'] == 'abc-123' } + ) + end + + it 'does not yield the omitted document' do + expect { |block| harvester.each_document(&block) }.to yield_successive_args({ 'layer_slug_s' => 'abc-321' }) + end + end + end + describe 'BlacklightResponseVersionFactory' do let(:version_class) { described_class::BlacklightResponseVersionFactory.call(json) }