Parse progressive input incrementally

- Replace raw input accumulation with row updates
- Match DSV header validation
- Add progressive data tests

Author: kojix2

lib/youplot/command.rb

@@ -112,25 +112,95 @@ def main(input)
     def main_progressive(input)
       output_data(input)
 
-      # FIXME
-      # Worked around the problem of not being able to draw
-      # plots when there is only one header line.
-      if @raw_data.nil?
-        @raw_data = String.new
-        if options[:headers]
-          @raw_data << input
-          return
-        end
-      end
-      @raw_data << input
+      row = parse_progressive_row(input)
+      return 0 if row.nil?
 
-      # FIXME
-      @data = parse_dsv(@raw_data)
+      @data = progressive_update_data(row)
+      return 0 if @data.nil?
 
       plot = create_plot
       output_plot_progressive(plot)
     end
 
+    def parse_progressive_row(input)
+      line = normalize_input_encoding!(input)
+
+      begin
+        row = CSV.parse_line(line, col_sep: options[:delimiter])
+      rescue CSV::MalformedCSVError => e
+        warn 'Failed to parse the text. '
+        warn 'Please try to set the correct character encoding with --encoding option.'
+        warn e.backtrace.grep(/youplot/).first
+        exit 1
+      rescue ArgumentError => e
+        warn 'Failed to parse the text. '
+        warn e.backtrace.grep(/youplot/).first
+        exit 1
+      end
+
+      return nil if row.nil? || row.empty? || row.all?(&:nil?)
+
+      row
+    end
+
+    def progressive_update_data(row)
+      init_progressive_state
+
+      return nil if consume_progressive_header?(row)
+
+      append_progressive_row(row)
+      progressive_data
+    end
+
+    def init_progressive_state
+      return if @progressive_initialized
+
+      @progressive_initialized = true
+      @progressive_headers = options[:headers] ? [] : nil
+      @progressive_series = []
+      @progressive_header_consumed = false
+      @progressive_row_count = 0
+    end
+
+    def consume_progressive_header?(row)
+      return false unless options[:headers]
+      return false if options[:transpose]
+      return false if @progressive_header_consumed
+
+      @progressive_headers = row
+      @progressive_header_consumed = true
+      true
+    end
+
+    def append_progressive_row(row)
+      if options[:headers] && options[:transpose]
+        @progressive_headers << row[0]
+        @progressive_series << row[1..-1]
+      elsif options[:transpose]
+        @progressive_series << row
+      else
+        append_progressive_columns(row)
+      end
+    end
+
+    def progressive_data
+      DSV.build_data(@progressive_headers, @progressive_series)
+    end
+
+    def append_progressive_columns(row)
+      if row.size > @progressive_series.size
+        (@progressive_series.size...row.size).each do |i|
+          @progressive_series[i] = Array.new(@progressive_row_count, nil)
+        end
+      end
+
+      0.upto(@progressive_series.size - 1) do |i|
+        @progressive_series[i] << row[i]
+      end
+
+      @progressive_row_count += 1
+    end
+
     def parse_dsv(input)
       # If encoding is specified, convert to UTF-8
       normalize_input_encoding!(input)

test/youplot/progressive_test.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require 'stringio'
+require_relative '../test_helper'
+
+class YouPlotProgressiveTest < Test::Unit::TestCase
+  def build_options(overrides = {})
+    out = StringIO.new
+    defaults = YouPlot::Options::DEFAULTS.merge(output: out, pass: false)
+    options_hash = defaults.merge(overrides)
+    YouPlot::Options.new(*YouPlot::Options.members.map { |k| options_hash[k] })
+  end
+
+  def build_data_with_progressive(command, lines)
+    data = nil
+    lines.each do |line|
+      row = command.send(:parse_progressive_row, line)
+      next if row.nil?
+
+      data = command.send(:progressive_update_data, row)
+    end
+    data
+  end
+
+  def suppress_stderr
+    stderr = $stderr
+    $stderr = StringIO.new
+    yield
+  ensure
+    $stderr = stderr
+  end
+
+  test 'progressive incremental parsing without headers matches DSV.parse' do
+    command = YouPlot::Command.new([])
+    command.options = build_options(delimiter: "\t", headers: false, transpose: false)
+
+    lines = %W[1\t2\n 3\n \n]
+    actual = build_data_with_progressive(command, lines)
+    expected = YouPlot::DSV.parse(lines.join, "\t", false, false)
+
+    assert_equal expected.headers, actual.headers
+    assert_equal expected.series, actual.series
+    assert_nil command.instance_variable_get(:@raw_data)
+  end
+
+  test 'progressive incremental parsing with headers waits for first data row' do
+    command = YouPlot::Command.new([])
+    command.options = build_options(delimiter: "\t", headers: true, transpose: false)
+
+    header_row = command.send(:parse_progressive_row, "h1\th2\n")
+    first = command.send(:progressive_update_data, header_row)
+    assert_nil first
+
+    data_row = command.send(:parse_progressive_row, "1\t2\n")
+    actual = command.send(:progressive_update_data, data_row)
+    expected = YouPlot::DSV.parse("h1\th2\n1\t2\n", "\t", true, false)
+
+    assert_equal expected.headers, actual.headers
+    assert_equal expected.series, actual.series
+  end
+
+  test 'progressive incremental parsing with headers and transpose matches DSV.parse' do
+    command = YouPlot::Command.new([])
+    command.options = build_options(delimiter: "\t", headers: true, transpose: true)
+
+    lines = %W[h1\t1\t2\n h2\t3\t4\n]
+    actual = build_data_with_progressive(command, lines)
+    expected = YouPlot::DSV.parse(lines.join, "\t", true, true)
+
+    assert_equal expected.headers, actual.headers
+    assert_equal expected.series, actual.series
+  end
+
+  test 'progressive with headers returns nil when headers are fewer than series' do
+    command = YouPlot::Command.new([])
+    command.options = build_options(delimiter: "\t", headers: true, transpose: false)
+
+    lines = %W[h1\n 1\t2\n]
+    actual = nil
+    expected = nil
+    suppress_stderr do
+      actual = build_data_with_progressive(command, lines)
+      expected = YouPlot::DSV.parse(lines.join, "\t", true, false)
+    end
+
+    assert_nil expected
+    assert_nil actual
+  end
+
+  test 'progressive with headers returns nil when headers are greater than series' do
+    command = YouPlot::Command.new([])
+    command.options = build_options(delimiter: "\t", headers: true, transpose: false)
+
+    lines = %W[h1\th2\n 1\n]
+    actual = nil
+    expected = nil
+    suppress_stderr do
+      actual = build_data_with_progressive(command, lines)
+      expected = YouPlot::DSV.parse(lines.join, "\t", true, false)
+    end
+
+    assert_nil expected
+    assert_nil actual
+  end
+end