rdcompiler.rb

# frozen_string_literal: true
#
# bitclust/rdcompiler.rb
#
# Copyright (C) 2006-2008 Minero Aoki
#
# This program is free software.
# You can distribute/modify this program under the Ruby License.
#

require 'bitclust/methodsignature'
require 'bitclust/lineinput'
require 'bitclust/htmlutils'
require 'bitclust/textutils'
require 'bitclust/messagecatalog'
require 'bitclust/syntax_highlighter'
require 'rouge'
require 'stringio'

module BitClust

  # Compiles doc into HTML.
  class RDCompiler

    include HTMLUtils
    include TextUtils
    include Translatable

    def initialize(urlmapper, hlevel = 1, opt = {})
      @urlmapper = urlmapper
      @catalog = opt[:catalog]
      @hlevel = hlevel
      @type = nil
      @method = nil
      @option = opt.dup
      init_message_catalog(@catalog || MessageCatalog.new({}, 'C'))
    end

    def compile(src)
      setup(src) {
        library_file
      }
    end

    def compile_function(f, opt = nil)
      @opt = opt
      @type = :function
      setup(f.source, f) {
        entry
      }
    ensure
      @opt = nil
    end

    # FIXME
    def compile_method(m, opt = nil)
      @opt = opt
      @type = :method
      @method = m
      setup(m.source, m) {
        entry
      }
    ensure
      @opt = nil
    end

    private

    def setup(src, entry = nil)
      @f = LineInput.new(StringIO.new(src), entry)
      @out = StringIO.new
      yield
      @out.string
    end

    def library_file
      while @f.next?
        case @f.peek
        when /\A---/
          entry_chunk
        when /\A=+/
          headline @f.gets || raise
        when /\A(\s+)\*\s/, /\A(\s+)\(\d+\)\s/
          @item_stack = []
          item_list(($1 || raise).size)
          raise "@item_stack should be empty. #{@item_stack.inspect}" unless @item_stack.empty?
        when %r<\A//emlist(?:\[(?:[^\[\]]+?)?\]\[\w+?\])?\{>
          emlist
        when /\A:\s/
          dlist
        when /\A\s+\S/
          list
        when /\A@see\b/
          # findings#1: doc/lib ページの @see もメソッドエントリと同じく
          # SEE_ALSO として解釈する（従来は段落テキストだった）
          see
        else
          if @f.peek&.strip&.empty?
            @f.gets
          else
            paragraph
          end
        end
      end
    end

    def entry
      while @f.next?
        entry_chunk
      end
    end

    # シグネチャ行に続くメソッド属性行({: ...}。直前のシグネチャ行に束縛)
    METHOD_ATTRIBUTE_LINE_RE = /\A\{:.*\}[ \t]*$/

    def entry_chunk
      @out.puts '<dl>' if @option[:force]
      first = true
      attrs = [] #: Array[String]
      while @f.next?
        if /\A---/ =~ @f.peek
          method_signature(@f.gets || raise, first)
          first = false
        elsif !first && METHOD_ATTRIBUTE_LINE_RE =~ @f.peek
          # メソッド属性行は本文には描画しない(undef のみ後でメッセージ)
          attrs.concat attribute_tokens(@f.gets)
        else
          break
        end
      end
      props = {} #: Hash[String?, String?]
      @f.while_match(/\A:/) do |line|
        k, v = line.sub(/\A:/, '').split(':', 2)
        props[k&.strip] = v&.strip
      end if @type == :method
      @out.puts %Q(<dd class="#{@type.to_s}-description">)
      undef_message if attrs.include?('undef')
      while @f.next?
        case @f.peek
        when /\A===+/
          headline @f.gets || raise
        when /\A==?/
          if @option[:force]
            break
          else
            raise "#{@type.to_s} entry includes headline: #{@f.peek.inspect}"
          end
        when /\A---/
          break
        when /\A(\s+)\*\s/, /\A(\s+)\(\d+\)\s/
          @item_stack = []
          item_list(($1 || raise).size)
          raise "@item_stack should be empty. #{@item_stack.inspect}" unless @item_stack.empty?
        when /\A:\s/
          dlist
        when %r<\A//emlist(?:\[(?:[^\[\]]+?)?\]\[\w+?\])?\{>
          emlist
        when /\A\s+\S/
          list
        when /\A@see\b/
          # findings#3: 無アンカーの /@see/ は行の途中に @see を含む
          # 本文行を吸ってしまう（@todo も同様）
          see
        when /\A@todo\b/
          todo
        when /\A@[a-z]/
          entry_info
        else
          if @f.peek&.strip&.empty?
            @f.gets
          else
            entry_paragraph
          end
        end
      end
      @out.puts '</dd>'
      @out.puts '</dl>' if @option[:force]
    end

    def headline(line)
      level = @hlevel + (line.slice(/\A=+/)&.size.to_i - 3)
      label = line.sub(/\A=+(\[a:(.*?)\])?/, '').strip
      frag = $2 if $2 and not ($2 || raise).empty?
      line h(level, escape_html(label), frag)
    end

    def h(level, label, frag = nil)
      name = frag ? "id='#{escape_html(frag)}'" : ""
      "<h#{level} #{name}>#{label}</h#{level}>"
    end

    ITEM_RE = /\A(\s+)(?:\*\s|\(\d+\))/

    def item_list(level = 0)
      open_tag = nil
      close_tag = nil
      case @f.peek
      when /\A(\s+)\*\s/
        open_tag  =  "<ul>"
        close_tag = "</ul>"
      when /\A(\s+)\(\d+\)\s/
        open_tag  = "<ol>"
        close_tag = "</ol>"
      end
      line open_tag
      @item_stack.push(close_tag)
      @f.while_match(ITEM_RE) do |line|
        string "<li>"
        @item_stack.push("</li>")
        string compile_text(line.sub(/\A(\s+)(?:\*|\(\d+\))/, '').strip)
        if /\A(\s+)(?!\*\s|\(\d+\))\S/ =~ @f.peek
          @f.while_match(/\A\s+(?!\*\s|\(\d+\))\S/) do |cont|
            nl
            string compile_text(cont.strip)
          end
        end
        if (m = ITEM_RE.match(@f.peek)) && level < (m[1] || raise).size
          item_list(($1 || raise).size)
          line @item_stack.pop # current level li
          break if ITEM_RE =~ @f.peek and level > ($1 || raise).size
        elsif m && level > (m[1] || raise).size
          line @item_stack.pop # current level li
          break
        else
          line @item_stack.pop # current level li
        end
      end
      line @item_stack.pop unless @item_stack.empty?
    end

    def dlist
      line '<dl>'
      while @f.next? and /\A:/ =~ @f.peek
        @f.while_match(/\A:/) do |line|
          line dt(compile_text(line.sub(/\A:/, '').strip))
        end
        dd_with_p
      end
      line '</dl>'
    end

    # empty lines separate paragraphs.
    def dd_with_p
      line '<dd>'
      while /\A(?:\s|\z)/ =~ @f.peek or %r!\A//emlist(?:\[(?:[^\[\]]+?)?\]\[\w+?\])?\{! =~ @f.peek
        case @f.peek
        when /\A$/
          @f.gets
        when  /\A[ \t]/
          line '<p>'
          line compile_text(text_node_from_lines(@f.span(/\A[ \t]/)))
          line '</p>'
        when %r!\A//emlist(?:\[(?:[^\[\]]+?)?\]\[\w+?\])?\{!
          emlist
        else
          raise 'must not happen'
        end
      end
      line '</dd>'
    end

    # empty lines do not separate paragraphs.
    def dd_without_p
      line '<dd>'
      while /\A[ \t]/ =~ @f.peek or %r!\A//emlist(?:\[(?:[^\[\]]+?)?\]\[\w+?\])?\{! =~ @f.peek
        case @f.peek
        when  /\A[ \t]/
          line compile_text(text_node_from_lines(@f.span(/\A[ \t]/)))
        when %r!\A//emlist(?:\[(?:[^\[\]]+?)?\]\[\w+?\])?\{!
          emlist
        end
      end
      line '</dd>'
    end

    def dt(s)
      "<dt>#{s}</dt>"
    end

    def stop_on_syntax_error?
      return true unless @option.key?(:stop_on_syntax_error)
      @option[:stop_on_syntax_error]
    end

    # lang 指定付きコードブロックのハイライト HTML を返す。
    # ruby(rb などの Rouge 上の alias を含む)は構文チェックを兼ねる
    # Ripper ベースの SyntaxHighlighter、その他の言語は Rouge を使う。
    # Rouge が知らない言語はエスケープのみ(従来この経路と ruby の構文
    # エラー時フォールバックはエスケープされずに出力されていたのを修正)
    def highlight_source(src, lang, caption)
      lexer = ::Rouge::Lexer.find(lang)
      if lexer && lexer.tag == 'ruby'
        begin
          filename = (caption&.size || 0) > 2 ? caption : @f.name or raise
          BitClust::SyntaxHighlighter.new(src, filename).highlight
        rescue BitClust::SyntaxHighlighter::Error => ex
          $stderr.puts ex.message
          exit(false) if stop_on_syntax_error?
          escape_html(src)
        end
      elsif lexer
        ::Rouge::Formatters::HTML.new.format(lexer.lex(src))
      else
        escape_html(src)
      end
    end

    def emlist
      command = @f.gets
      if %r!\A//emlist\[(?<caption>[^\[\]]+?)?\]\[(?<lang>\w+?)\]! =~ command
        # @type var caption: String?
        # @type var lang: String
        # caption は pre 内の absolute 配置ではなく、pre の上に密着した
        # タブとして描画する(pre 内だと編集・貼り付けで先頭行と重なる)
        line "<span class=\"caption\">#{escape_html(caption)}</span>" if caption
        line "<pre class=\"highlight #{lang}\">"
        line "<code>"
        src = +""
        @f.until_terminator(%r<\A//\}>) do |line|
          src << line
        end
        string highlight_source(src, lang, caption)
        line '</code></pre>'
      else
        line '<pre>'
        @f.until_terminator(%r<\A//\}>) do |line|
          line escape_html(line.rstrip)
        end
        line '</pre>'
      end
    end

    def list
      lines = unindent_block(canonicalize(@f.break(/\A\S/)))
      while lines.last&.empty?
        lines.pop
      end
      line '<pre>'
      lines.each do |line|
        line escape_html(line)
      end
      line '</pre>'
    end

    def canonicalize(lines)
      lines.map {|line| detab(line.rstrip) }
    end

    def paragraph
      line '<p>'
      line compile_text(text_node_from_lines(read_paragraph(@f)))
      line '</p>'
    end

    def read_paragraph(f)
      f.span(%r<\A(?!---|=|//emlist(?:\[(?:[^\[\]]+?)?\]\[\w+?\])?\{)\S>)
    end

    def see
      header = @f.gets or raise
      header.slice!(/\A\@\w+/)
      body = [header] + @f.span(/\A\s+\S/)
      line '<p>'
      line '[SEE_ALSO] ' + compile_text(text_node_from_lines(body))
      line '</p>'
    end

    def todo
      header = @f.gets or raise
      header.slice!(/\A\@\w+/)
      body = header
      line '<p class="todo">'
      line '[TODO]' + body
      line '</p>'
    end

    def attribute_tokens(attr_line)
      (attr_line || '')[/\A\{:(.*)\}/, 1].to_s.strip.split(/\s+/)
    end

    # {: undef} 属性付きエントリのメッセージ。ページを見に来た人向け
    # (statichtml は undefined エントリを skip するので server 等の動的経路用)。
    # nomethod は説明が本文に書かれている前提のマーカーなので何も描画しない
    def undef_message
      line '<p>'
      line 'このメソッドは定義されていません。'
      line '</p>'
    end

    def entry_info
      line '<dl>'
      while @f.next? and /\A\@(?!see)\w+|\A$/ =~ @f.peek
        header = @f.gets or raise
        next if /\A$/ =~ header
        cmd = header.slice!(/\A\@\w+/) or raise
        @f.ungets(header)
        case cmd
        when '@param', '@arg'
          name = header.slice!(/\A\s*\w+/) || '?'
          line "<dt class='#{@type.to_s}-param'>[PARAM] #{escape_html(name.strip)}:</dt>"
        when '@raise'
          ex = header.slice!(/\A\s*[\w:]+/) || '?'
          line "<dt>[EXCEPTION] #{escape_html(ex.strip)}:</dt>"
        when '@return'
          line "<dt>[RETURN]</dt>"
        else
          line "<dt>[UNKNOWN_META_INFO] #{escape_html(cmd)}:</dt>"
        end
        dd_without_p
      end
      line '</dl>'
    end

    # FIXME: parse @param, @return, ...
    def entry_paragraph
      line '<p>'
      line compile_text(text_node_from_lines(read_entry_paragraph(@f)))
      line '</p>'
    end

    def read_entry_paragraph(f)
      f.span(%r<\A(?!---|=|//emlist(?:\[(?:[^\[\]]+?)?\]\[\w+?\])?\{|@[a-z])\S>)
    end

    def method_signature(sig_line, first)
      # FIXME: check parameters, types, etc.
      sig = MethodSignature.parse(sig_line)
      string %Q(<dt class="method-heading")
      string %Q( id="#{@method.index_id}") if first
      string '><code>'
      string @method.klass.name + @method.typemark if @opt
      string escape_html(sig.friendly_string)
      string '</code>'
      if first
        string '<span class="permalink">['
        string a_href(@urlmapper.method_url(methodid2specstring(@method.id)), "permalink")
        string ']['
        string rdoc_link(@method.id, @option[:database].properties["version"])
        if @option[:edit_base_url] && @method.source_location
          string ']['
          # @type var urlmapper: ::BitClust::Subcommands::StatichtmlCommand::URLMapperEx
          urlmapper = _ = @urlmapper
          string a_href(urlmapper.edit_url(@method.source_location), 'edit')
        end
        string ']</span>'
      end
      if @method and not @method.defined?
        line %Q( <span class="kindinfo">[#{@method.kind} by #{library_link(@method.library.name)}]</span>)
      end
      line '</dt>'
    end

    BracketLink = /\[\[[\w-]+?:[!-~]+?(?:\[\] )?\]\]/n
    # BitClust には ISO/JIS へのリンク機能がないため、[[ISO:8601]]/[[JIS:X 0301]] は
    # リンクにせず平文で表示する (rurema/bitclust#236)。引数に空白を含む JIS も拾う。
    StandardRef = /\[\[(?:ISO|JIS):[!-~][!-~ ]*?\]\]/n
    NeedESC = /[&"<>]/

    def compile_text(str)
      escape_table = HTMLUtils::ESC
      str.gsub(/(#{NeedESC})|(#{StandardRef})|(#{BracketLink})/o) {
        # @type var char: '&' | '"' | '<' | '>'
        if    char = _ = $1 then escape_table[char]
        elsif tok  = $2 then standard_ref(tok[2..-3] || raise)
        elsif tok  = $3 then bracket_link(tok[2..-3] || raise)
        else
          raise 'must not happen'
        end
      }
    end

    def standard_ref(link)
      # 例: "ISO:8601" -> "ISO 8601", "JIS:X 0301" -> "JIS X 0301"
      type, _arg = link.split(':', 2)
      arg = _arg&.strip or raise
      escape_html("#{type} #{arg}")
    end

    def bracket_link(link, label = nil, frag = nil)
      type, _arg = link.split(':', 2)
      arg = _arg&.rstrip or raise
      case type
      when 'lib'
        protect(link) {
          case arg
          when '/', '_index'
            label = _('Library Index')
          when '_builtin'
            label = _('Builtin Library')
          end
          library_link(arg, label, frag)
        }
      when 'c'
        protect(link) { class_link(arg, label, frag) }
      when 'm'
        protect(link) { method_link(complete_spec(arg), label || arg, frag) }
      when 'f'
        protect(link) {
          case arg
          when '/', '_index'
            arg, label = '', _('Function Index')
          end
          function_link(arg, label || arg, frag)
        }
      when 'd'
        protect(link) { document_link(arg, label, frag) }
      when 'ref'
        protect(link) { reference_link(arg) }
      when 'url'
        direct_url(arg)
      when 'man'
        man_link(arg)
      when 'rfc', 'RFC'
        rfc_link(arg)
      when 'ruby-list', 'ruby-dev', 'ruby-ext', 'ruby-talk', 'ruby-core'
        blade_link(type, arg)
      when 'feature', 'bug', 'misc'
        bugs_link(type, arg)
      else
        "[[#{escape_html(link)}]]"
      end
    end

    def protect(src)
      yield
    rescue => err
      %Q(<span class="compileerror">[[compile error: #{escape_html(err.message)}: #{escape_html(src)}]]</span>)
    end

    def direct_url(url)
      %Q(<a class="external" href="#{escape_html(url)}">#{escape_html(url)}</a>)
    end

    def reference_link(arg)
      case arg
      when /(\w+):(.*)\#(\w+)\z/
        # @type var type: String
        # @type var name: String
        # @type var frag: String
        type, name, frag = $1, $2, $3
        # @type var title: String
        # @type var t: String
        # @type var id: String
        case type
        when 'lib'
          title, t, id = name, LibraryEntry.type_id.to_s, name
        when 'c'
          title, t, id = name, ClassEntry.type_id.to_s,   name
        when 'm'
          title, t, id = name, MethodEntry.type_id.to_s,  name
        when 'd'
          title, t, id = @option[:database].get_doc(name).title, DocEntry.type_id.to_s, name
        else
          raise "must not happen"
        end
        label = @option[:database].refs[t, id, frag]
        label = title + '/' + label if label and name
        bracket_link("#{type}:#{name}", label, frag)
      when /\A(\w+)\z/
        e = @option[:entry]
        frag = $1 || raise
        type = e.type_id.to_s
        label = @option[:database].refs[type, e.name, frag] || frag
        a_href('#' + frag, label)
      else
        raise "must not happen"
      end
    end

    BLADE_URL = 'https://blade.ruby-lang.org/%s/%s'

    def blade_link(ml, num)
      url = sprintf(BLADE_URL, ml, num)
      %Q(<a class="external" href="#{escape_html(url)}">[#{escape_html("#{ml}:#{num}")}]</a>)
    end

    RFC_URL = 'https://tools.ietf.org/html/rfc%s'

    def rfc_link(num)
      url = sprintf(RFC_URL, num)
      %Q(<a class="external" href="#{escape_html(url)}">[RFC#{escape_html(num)}]</a>)
    end

    opengroup_url = 'http://www.opengroup.org/onlinepubs/009695399'
    MAN_CMD_URL = "#{opengroup_url}/utilities/%s.html"
    MAN_FCN_URL = "#{opengroup_url}/functions/%s.html"
    MAN_HEADER_URL = "#{opengroup_url}/basedefs/%s.html"
    MAN_LINUX_URL = "http://man7.org/linux/man-pages/man%1$s/%2$s.%1$s.html"
    MAN_FREEBSD_URL = "http://www.freebsd.org/cgi/man.cgi?query=%2$s&sektion=%1$s&manpath=FreeBSD+9.0-RELEASE"

    def man_url(section, page)
      case section
      when "1"
        sprintf(MAN_CMD_URL, page)
      when "2", "3"
        sprintf(MAN_FCN_URL, page)
      when "header"
        sprintf(MAN_HEADER_URL, page)
      when /\A([23457])linux\Z/
        sprintf(MAN_LINUX_URL, $1, page)
      when /\A([1-9])freebsd\Z/
        sprintf(MAN_FREEBSD_URL, $1, page)
      else
        nil
      end
    end

    def man_link(spec)
      m = /([\w\.\/]+)\((\w+)\)/.match(spec) or return escape_html(spec)
      url = man_url(m[2], escape_html(m[1] || raise)) or return escape_html(spec)
      %Q(<a class="external" href="#{escape_html(url)}">#{escape_html("#{m[1]}(#{m[2]})")}</a>)
    end

    BUGS_URL = "https://bugs.ruby-lang.org/issues/%s"

    def bugs_link(type, id)
      url = sprintf(BUGS_URL, id)
      %Q(<a class="external" href="#{escape_html(url)}">[#{type}##{id}]</a>)
    end

    def rdoc_url(method_id, version)
      cname, tmark, mname, _libname = methodid2specparts(method_id)
      tchar = typemark2char(tmark) == 'i' ? 'i' : 'c'
      cname = cname.split(".").first || raise
      cname = cname.gsub('::', '/')
      id = "method-#{tchar}-#{encodename_rdocurl(mname)}"

      "https://docs.ruby-lang.org/en/#{version}/#{cname}.html##{id}"
    end

    def rdoc_link(method_id, version)
      a_href(rdoc_url(method_id, version), "rdoc")
    end

    def complete_spec(spec0)
      case spec0
      when /\A\$/
        "Kernel#{spec0}"
      else
        spec0
      end
    end

    def seems_code(text)
      # FIXME
      escape_html(text)
    end

    def string(str)
      @out.print str
    end

    def line(str)
      @out.puts str
    end

    def nl
      @out.puts
    end

    def text_node_from_lines(lines)
      lines.map(&:strip).join("\n").gsub(/(\P{ascii})\n(\P{ascii})/) do
        "#{::Regexp.last_match(1)}#{::Regexp.last_match(2)}"
      end
    end
  end

end