Expose harvested documents via each_document

README.md

@@ -197,6 +197,17 @@ Crawl delays can be configured (in seconds) either globally for all sites or on
 
 Solr's commitWithin option can be configured (in milliseconds) by passing a value under the commit_within key.
 
+#### Harvesting Documents Without Indexing Them
+
+The harvester exposes the documents it harvests as an enumerable, so you can do something with them other than index them into Solr -- writing them to disk to contribute them to your own OpenGeoMetadata repository, for example. The documents yielded have already been through the configured document transformer.
+
+```ruby
+harvester = GeoCombine::GeoBlacklightHarvester.new(:SITE1)
+harvester.each_document do |document|
+  File.write("#{document['id']}.json", JSON.pretty_generate(document))
+end
+```
+
 #### Transforming Documents
 
 You may need to transform documents that are harvested for various purposes (removing fields, adding fields, omitting a document all together, etc). You can configure some ruby code (a proc) that will take the document in, transform it, and return the transformed document. By default the indexer will remove the `score`, `timestamp`, and `_version_` fields from the documents harvested. If you provide your own transformer, you'll likely want to remove these fields in addition to the other transformations you provide.

lib/geo_combine/geo_blacklight_harvester.rb

@@ -55,25 +55,25 @@ def initialize(site_key, logger: GeoCombine::Logger.logger)
       raise ArgumentError, "Site key #{@site_key.inspect} is not configured for #{self.class.name}" unless @site
     end
 
+    # Index the documents harvested from the site into Solr
     def index
-      @logger.debug "fetching page 1 @ #{base_url}&page=1"
-      response = JSON.parse(Net::HTTP.get(URI("#{base_url}&page=1")))
-      response_class = BlacklightResponseVersionFactory.call(response)
-
-      response_class.new(response:, base_url:, logger: @logger).documents.each do |docs|
-        docs.map! do |document|
-          self.class.document_transformer&.call(document)
-        end.compact
-
-        @logger.debug "adding #{docs.count} documents to solr"
+      each_page do |documents|
+        @logger.debug "adding #{documents.count} documents to solr"
         solr_connection.update params: { commitWithin: commit_within, overwrite: true },
-                               data: docs.to_json,
+                               data: documents.to_json,
                                headers: { 'Content-Type' => 'application/json' }
-
-        sleep(crawl_delay.to_i) if crawl_delay
       end
     end
 
+    # Enumerable of the documents harvested from the site, for passing to an
+    # indexer or doing something else with them (e.g. writing them to disk).
+    # Documents have already been through the configured document transformer.
+    def each_document(&block)
+      return to_enum(:each_document) unless block_given?
+
+      each_page { |documents| documents.each(&block) }
+    end
+
     ##
     # A "factory" class to determine the blacklight response version to use
     class BlacklightResponseVersionFactory
@@ -188,6 +188,21 @@ def documents_from_urls(urls)
 
     private
 
+    # Enumerable of pages of transformed documents harvested from the site
+    def each_page
+      return to_enum(:each_page) unless block_given?
+
+      @logger.debug "fetching page 1 @ #{base_url}&page=1"
+      response = JSON.parse(Net::HTTP.get(URI("#{base_url}&page=1")))
+      response_class = BlacklightResponseVersionFactory.call(response)
+
+      response_class.new(response:, base_url:, logger: @logger).documents.each do |documents|
+        yield documents.map { |document| self.class.document_transformer&.call(document) }.compact
+
+        sleep(crawl_delay.to_i) if crawl_delay
+      end
+    end
+
     def base_url
       "#{site[:host]}?#{default_params.to_query}"
     end

spec/lib/geo_combine/geo_blacklight_harvester_spec.rb

@@ -120,6 +120,40 @@
     end
   end
 
+  describe '#each_document' do
+    before do
+      expect(Net::HTTP).to receive(:get).with(
+        URI('https://example.com?f%5Bdct_provenance_s%5D%5B%5D=INSTITUTION&format=json&per_page=100&page=1')
+      ).and_return(stub_json_response)
+    end
+
+    let(:docs) { [{ 'layer_slug_s' => 'abc-123', 'score' => 0.1 }, { 'layer_slug_s' => 'abc-321' }] }
+    let(:transformed_docs) { [{ 'layer_slug_s' => 'abc-123' }, { 'layer_slug_s' => 'abc-321' }] }
+    let(:stub_json_response) do
+      { response: { docs:, pages: { current_page: 1, total_pages: 1 } } }.to_json
+    end
+
+    it 'yields each transformed document' do
+      expect { |block| harvester.each_document(&block) }.to yield_successive_args(*transformed_docs)
+    end
+
+    it 'returns an enumerator when no block is given' do
+      expect(harvester.each_document.to_a).to eq(transformed_docs)
+    end
+
+    context 'when the document transformer omits a document' do
+      before do
+        allow(described_class).to receive(:document_transformer).and_return(
+          ->(document) { document unless document['layer_slug_s'] == 'abc-123' }
+        )
+      end
+
+      it 'does not yield the omitted document' do
+        expect { |block| harvester.each_document(&block) }.to yield_successive_args({ 'layer_slug_s' => 'abc-321' })
+      end
+    end
+  end
+
   describe 'BlacklightResponseVersionFactory' do
     let(:version_class) { described_class::BlacklightResponseVersionFactory.call(json) }