New approach with code generated paginator functions

Author: Matt Muller

gems/smithy-client/lib/smithy-client.rb

@@ -19,7 +19,6 @@
 require_relative 'smithy-client/managed_file'
 require_relative 'smithy-client/networking_error'
 require_relative 'smithy-client/pageable_output'
-require_relative 'smithy-client/paginator'
 require_relative 'smithy-client/param_converter'
 require_relative 'smithy-client/param_validator'
 require_relative 'smithy-client/plugin'

gems/smithy-client/lib/smithy-client/pageable_output.rb

@@ -51,33 +51,34 @@ module Client
     #     end
     #
     module PageableOutput
-      def self.extended(base)
-        base.instance_variable_set(:@last_page, nil)
-      end
-
       # @return [Paginator]
-      attr_accessor :pager
+      attr_accessor :paginator
 
       # Returns `true` if there are no more results. Calling {#next_page}
       # when this method returns `false` will raise an error.
       # @return [Boolean]
       def last_page?
-        @last_page = !@pager.truncated?(self) if @last_page.nil?
-        !!@last_page
+        return @last_page if @last_page
+
+        @last_page = !truncated?
       end
 
       # Returns `true` if there are more results. Calling {#next_page} will
       # return the next response.
       # @return [Boolean]
       def next_page?
-        !last_page?
+        return @next_page if @next_page
+
+        @next_page = truncated?
       end
 
-      # @return [Output]
+      # @param [Hash] params A hash of additional request params.
+      # @return [Output] Returns the next page of results.
       def next_page(params = {})
         raise LastPageError, self if last_page?
 
-        next_response(params)
+        params = next_page_params(params)
+        context.client.send(context.operation_name, params)
       end
 
       # Yields the current and each following output to the given block.
@@ -97,33 +98,30 @@ def each_page(&)
       # @return [Enumerable, nil] Returns a new Enumerable if no block is given.
       def each_item(&)
         output = self
-        @pager.items(output).each(&)
+        @paginator.items(output.data).each(&)
         until output.last_page?
           output = output.next_page
-          @pager.items(output).each(&)
+          @paginator.items(output.data).each(&)
         end
       end
 
       private
 
-      # @param [Hash] params A hash of additional request params.
-      # @return [Output] Returns the next page of results.
-      def next_response(params)
-        params = next_page_params(params)
-        context.client.send(context.operation_name, params)
+      def truncated?
+        next_t = @paginator.next_tokens(data)
+        !(next_t.empty? || next_t == @paginator.prev_tokens(context.params))
       end
 
       # @param [Hash] params A hash of additional request params to
       #   merge into the next page request.
       # @return [Hash] Returns the hash of request parameters for the
       #   next page, merging any given params.
       def next_page_params(params)
+        prev_tokens = @paginator.prev_tokens(context.params)
         # Remove all previous tokens from original params
         # Sometimes a token can be nil and merge would not include it.
-        prev_tokens = @pager.prev_tokens(self)
-        params_without_tokens = context[:original_params].reject { |k, _v| prev_tokens.include?(k) }
-        params_without_tokens.merge!(@pager.next_tokens(self).merge(params))
-        params_without_tokens
+        new_params = context.params.except(*prev_tokens)
+        new_params.merge!(@paginator.next_tokens(data).merge(params))
       end
     end
   end

gems/smithy-client/lib/smithy-client/paginator.rb

@@ -8,27 +8,21 @@ class PageableOutput < Plugin
         # @api private
         class Handler < Client::Handler
           def call(context)
-            context[:original_params] = context.params
             output = @handler.call(context)
             output.extend(Client::PageableOutput)
-            trait = paginated_trait(context)
-            output.pager = Paginator.new(
-              input_token: trait['inputToken'],
-              output_token: trait['outputToken'],
-              items: trait['items'],
-              page_size: trait['pageSize']
-            )
+            output.paginator = context.operation[:paginator] || NullPaginator.new
             output
           end
 
-          private
+          # @api private
+          class NullPaginator
+            def next_tokens(_data)
+              {}
+            end
 
-          def paginated_trait(context)
-            service = context.config.service
-            operation = context.operation
-            service
-              .traits.fetch('smithy.api#paginated', {})
-              .merge(operation.traits.fetch('smithy.api#paginated', {}))
+            def prev_tokens(_params)
+              {}
+            end
           end
         end
 

gems/smithy-client/lib/smithy-client/plugins/pageable_output.rb

@@ -9,13 +9,25 @@ class Shape
         def initialize(options = {})
           @id = options[:id]
           @traits = options[:traits] || {}
+          @metadata = {}
         end
 
         # @return [String, nil] Absolute shape ID from model
         attr_accessor :id
 
         # @return [Hash<String, Object>]
         attr_accessor :traits
+
+        # @return [Hash<Object, Object>]
+        def [](key)
+          @metadata[key]
+        end
+
+        # @param [Object] key
+        # @param [Object] value
+        def []=(key, value)
+          @metadata[key] = value
+        end
       end
 
       # Represents an aggregate shape that has members.

gems/smithy-schema/lib/smithy-schema/shapes.rb

@@ -6,6 +6,8 @@ module Smithy
 
         attr_accessor id: String?
         attr_accessor traits: Hash[String, untyped]
+        def []: (untyped) -> untyped
+        def []=: (untyped, untyped) -> void
       end
 
       class Structure < Shape

gems/smithy-schema/sig/smithy-schema/shapes.rbs

@@ -26,6 +26,12 @@ module Shapes
             subject = Shape.new(traits: { 'trait' => 'value' })
             expect(subject.traits).to eq({ 'trait' => 'value' })
           end
+
+          it 'can get and set metadata' do
+            subject = Shape.new
+            subject[:foo] = 'bar'
+            expect(subject[:foo]).to eq('bar')
+          end
         end
       end
 

gems/smithy-schema/spec/smithy-schema/shapes_spec.rb

@@ -49,6 +49,7 @@ def source_files
           e.yield "lib/#{@gem_name}/errors.rb", Views::Client::Errors.new(@plan).render
           e.yield "lib/#{@gem_name}/endpoint_parameters.rb", Views::Client::EndpointParameters.new(@plan).render
           e.yield "lib/#{@gem_name}/endpoint_provider.rb", Views::Client::EndpointProvider.new(@plan).render
+          e.yield "lib/#{@gem_name}/paginators.rb", Views::Client::Paginators.new(@plan).render
           code_generated_plugins.each { |path, plugin| e.yield path, plugin.source }
           e.yield "lib/#{@gem_name}/types.rb", Views::Client::Types.new(@plan).render
           e.yield "lib/#{@gem_name}/schema.rb", Views::Client::Schema.new(@plan).render

gems/smithy/lib/smithy/generators/client.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+# This is generated code!
+
+module <%= module_name %>
+  # @api private
+  module Paginators
+
+<% paginators.each do |paginator| -%>
+    # @api private
+    class <%= paginator.name %>
+      def next_tokens(data)
+<% paginator.next_tokens_code.each do |line| -%>
+        <%= line %>
+<% end -%>
+      end
+
+      def prev_tokens(params)
+<% paginator.prev_tokens_code.each do |line| -%>
+        <%= line %>
+<% end -%>
+      end
+<%- if paginator.items? -%>
+
+      def items(data)
+<% paginator.items_code.each do |line| -%>
+        <%= line %>
+<% end -%>
+      end
+<% end -%>
+    end
+
+<% end -%>
+  end
+end

gems/smithy/lib/smithy/templates/client/paginators.erb

@@ -26,10 +26,12 @@ module <%= module_name %>
     SERVICE = ServiceShape.new do |service|
       service.id = "<%= service_shape.id %>"
       service.name = "<%= service_shape.name %>"
+<% if service_shape.version? -%>
       service.version = "<%= service_shape.version %>"
+<% end -%>
       service.traits = <%= service_shape.traits %>
 <% operation_shapes.each do |shape| -%>
-      service.add_operation(:<%= shape.to_underscore %>, OperationShape.new do |operation|
+      service.add_operation(:<%= shape.symbol_name %>, OperationShape.new do |operation|
         operation.id = "<%= shape.id %>"
         operation.name = "<%= shape.name %>"
         operation.input = <%= shape.input %>
@@ -38,6 +40,9 @@ module <%= module_name %>
 <% shape.errors.each do |err| -%>
         operation.errors << <%= err %>
 <% end -%>
+<% if shape.paginated? -%>
+        operation[:paginator] = <%= shape.paginator %>
+<% end %>
       end)
 <% end -%>
     end