/* Package parser implements parser for markdown text that generates AST (abstract syntax tree). */ package parser import ( "bytes" "fmt" "strconv" "strings" "github.com/gomarkdown/markdown/ast" ) // Extensions is a bitmask of enabled parser extensions. type Extensions int // Bit flags representing markdown parsing extensions. // Use | (or) to specify multiple extensions. const ( NoExtensions Extensions = 0 NoIntraEmphasis Extensions = 1 << iota // Ignore emphasis markers inside words Tables // Parse tables FencedCode // Parse fenced code blocks Autolink // Detect embedded URLs that are not explicitly marked Strikethrough // Strikethrough text using ~~test~~ LaxHTMLBlocks // Loosen up HTML block parsing rules SpaceHeadings // Be strict about prefix heading rules HardLineBreak // Translate newlines into line breaks NonBlockingSpace // Translate backspace spaces into line non-blocking spaces TabSizeEight // Expand tabs to eight spaces instead of four Footnotes // Pandoc-style footnotes NoEmptyLineBeforeBlock // No need to insert an empty line to start a (code, quote, ordered list, unordered list) block HeadingIDs // specify heading IDs with {#id} Titleblock // Titleblock ala pandoc AutoHeadingIDs // Create the heading ID from the text BackslashLineBreak // Translate trailing backslashes into line breaks DefinitionLists // Parse definition lists MathJax // Parse MathJax OrderedListStart // Keep track of the first number used when starting an ordered list. Attributes // Block Attributes SuperSubscript // Super- and subscript support: 2^10^, H~2~O. EmptyLinesBreakList // 2 empty lines break out of list Includes // Support including other files. Mmark // Support Mmark syntax, see https://mmark.nl/syntax CommonExtensions Extensions = NoIntraEmphasis | Tables | FencedCode | Autolink | Strikethrough | SpaceHeadings | HeadingIDs | BackslashLineBreak | DefinitionLists | MathJax ) // The size of a tab stop. const ( tabSizeDefault = 4 tabSizeDouble = 8 ) // for each character that triggers a response when parsing inline data. type inlineParser func(p *Parser, data []byte, offset int) (int, ast.Node) // ReferenceOverrideFunc is expected to be called with a reference string and // return either a valid Reference type that the reference string maps to or // nil. If overridden is false, the default reference logic will be executed. // See the documentation in Options for more details on use-case. type ReferenceOverrideFunc func(reference string) (ref *Reference, overridden bool) // Parser is a type that holds extensions and the runtime state used by // Parse, and the renderer. You can not use it directly, construct it with New. type Parser struct { // ReferenceOverride is an optional function callback that is called every // time a reference is resolved. It can be set before starting parsing. // // In Markdown, the link reference syntax can be made to resolve a link to // a reference instead of an inline URL, in one of the following ways: // // * [link text][refid] // * [refid][] // // Usually, the refid is defined at the bottom of the Markdown document. If // this override function is provided, the refid is passed to the override // function first, before consulting the defined refids at the bottom. If // the override function indicates an override did not occur, the refids at // the bottom will be used to fill in the link details. ReferenceOverride ReferenceOverrideFunc // IsSafeURLOverride allows overriding the default URL matcher. URL is // safe if the overriding function returns true. Can be used to extend // the default list of safe URLs. IsSafeURLOverride func(url []byte) bool Opts Options // after parsing, this is AST root of parsed markdown text Doc ast.Node extensions Extensions refs map[string]*reference refsRecord map[string]struct{} inlineCallback [256]inlineParser nesting int maxNesting int insideLink bool indexCnt int // incremented after every index // Footnotes need to be ordered as well as available to quickly check for // presence. If a ref is also a footnote, it's stored both in refs and here // in notes. Slice is nil if footnotes not enabled. notes []*reference tip ast.Node // = doc oldTip ast.Node lastMatchedContainer ast.Node // = doc allClosed bool // Attributes are attached to block level elements. attr *ast.Attribute includeStack *incStack // collect headings where we auto-generated id so that we can // ensure they are unique at the end allHeadingsWithAutoID []*ast.Heading } // New creates a markdown parser with CommonExtensions. // // You can then call `doc := p.Parse(markdown)` to parse markdown document // and `markdown.Render(doc, renderer)` to convert it to another format with // a renderer. func New() *Parser { return NewWithExtensions(CommonExtensions) } // NewWithExtensions creates a markdown parser with given extensions. func NewWithExtensions(extension Extensions) *Parser { p := Parser{ refs: make(map[string]*reference), refsRecord: make(map[string]struct{}), maxNesting: 16, insideLink: false, Doc: &ast.Document{}, extensions: extension, allClosed: true, includeStack: newIncStack(), } p.tip = p.Doc p.oldTip = p.Doc p.lastMatchedContainer = p.Doc p.inlineCallback[' '] = maybeLineBreak p.inlineCallback['*'] = emphasis p.inlineCallback['_'] = emphasis if p.extensions&Strikethrough != 0 { p.inlineCallback['~'] = emphasis } p.inlineCallback['`'] = codeSpan p.inlineCallback['\n'] = lineBreak p.inlineCallback['['] = link p.inlineCallback['<'] = leftAngle p.inlineCallback['\\'] = escape p.inlineCallback['&'] = entity p.inlineCallback['!'] = maybeImage if p.extensions&Mmark != 0 { p.inlineCallback['('] = maybeShortRefOrIndex } p.inlineCallback['^'] = maybeInlineFootnoteOrSuper if p.extensions&Autolink != 0 { p.inlineCallback['h'] = maybeAutoLink p.inlineCallback['m'] = maybeAutoLink p.inlineCallback['f'] = maybeAutoLink p.inlineCallback['H'] = maybeAutoLink p.inlineCallback['M'] = maybeAutoLink p.inlineCallback['F'] = maybeAutoLink } if p.extensions&MathJax != 0 { p.inlineCallback['$'] = math } return &p } func (p *Parser) getRef(refid string) (ref *reference, found bool) { if p.ReferenceOverride != nil { r, overridden := p.ReferenceOverride(refid) if overridden { if r == nil { return nil, false } return &reference{ link: []byte(r.Link), title: []byte(r.Title), noteID: 0, hasBlock: false, text: []byte(r.Text)}, true } } // refs are case insensitive ref, found = p.refs[strings.ToLower(refid)] return ref, found } func (p *Parser) isFootnote(ref *reference) bool { _, ok := p.refsRecord[string(ref.link)] return ok } func (p *Parser) finalize(block ast.Node) { p.tip = block.GetParent() } func (p *Parser) addChild(node ast.Node) ast.Node { for !canNodeContain(p.tip, node) { p.finalize(p.tip) } ast.AppendChild(p.tip, node) p.tip = node return node } func canNodeContain(n ast.Node, v ast.Node) bool { switch n.(type) { case *ast.List: return isListItem(v) case *ast.Document, *ast.BlockQuote, *ast.Aside, *ast.ListItem, *ast.CaptionFigure: return !isListItem(v) case *ast.Table: switch v.(type) { case *ast.TableHeader, *ast.TableBody, *ast.TableFooter: return true default: return false } case *ast.TableHeader, *ast.TableBody, *ast.TableFooter: _, ok := v.(*ast.TableRow) return ok case *ast.TableRow: _, ok := v.(*ast.TableCell) return ok } return false } func (p *Parser) closeUnmatchedBlocks() { if p.allClosed { return } for p.oldTip != p.lastMatchedContainer { parent := p.oldTip.GetParent() p.finalize(p.oldTip) p.oldTip = parent } p.allClosed = true } // Reference represents the details of a link. // See the documentation in Options for more details on use-case. type Reference struct { // Link is usually the URL the reference points to. Link string // Title is the alternate text describing the link in more detail. Title string // Text is the optional text to override the ref with if the syntax used was // [refid][] Text string } // Parse generates AST (abstract syntax tree) representing markdown document. // // The result is a root of the tree whose underlying type is *ast.Document // // You can then convert AST to html using html.Renderer, to some other format // using a custom renderer or transform the tree. func (p *Parser) Parse(input []byte) ast.Node { p.block(input) // Walk the tree and finish up some of unfinished blocks for p.tip != nil { p.finalize(p.tip) } // Walk the tree again and process inline markdown in each block ast.WalkFunc(p.Doc, func(node ast.Node, entering bool) ast.WalkStatus { switch node.(type) { case *ast.Paragraph, *ast.Heading, *ast.TableCell: p.Inline(node, node.AsContainer().Content) node.AsContainer().Content = nil } return ast.GoToNext }) if p.Opts.Flags&SkipFootnoteList == 0 { p.parseRefsToAST() } // ensure HeadingIDs generated with AutoHeadingIDs are unique // this is delayed here (as opposed to done when we create the id) // so that we can preserve more original ids when there are conflicts taken := map[string]bool{} for _, h := range p.allHeadingsWithAutoID { id := h.HeadingID if id == "" { continue } n := 0 for taken[id] { n++ id = h.HeadingID + "-" + strconv.Itoa(n) } h.HeadingID = id taken[id] = true } return p.Doc } func (p *Parser) parseRefsToAST() { if p.extensions&Footnotes == 0 || len(p.notes) == 0 { return } p.tip = p.Doc list := &ast.List{ IsFootnotesList: true, ListFlags: ast.ListTypeOrdered, } p.addBlock(&ast.Footnotes{}) block := p.addBlock(list) flags := ast.ListItemBeginningOfList // Note: this loop is intentionally explicit, not range-form. This is // because the body of the loop will append nested footnotes to p.notes and // we need to process those late additions. Range form would only walk over // the fixed initial set. for i := 0; i < len(p.notes); i++ { ref := p.notes[i] p.addChild(ref.footnote) block := ref.footnote listItem := block.(*ast.ListItem) listItem.ListFlags = flags | ast.ListTypeOrdered listItem.RefLink = ref.link if ref.hasBlock { flags |= ast.ListItemContainsBlock p.block(ref.title) } else { p.Inline(block, ref.title) } flags &^= ast.ListItemBeginningOfList | ast.ListItemContainsBlock } above := list.Parent finalizeList(list) p.tip = above ast.WalkFunc(block, func(node ast.Node, entering bool) ast.WalkStatus { switch node.(type) { case *ast.Paragraph, *ast.Heading: p.Inline(node, node.AsContainer().Content) node.AsContainer().Content = nil } return ast.GoToNext }) } // // Link references // // This section implements support for references that (usually) appear // as footnotes in a document, and can be referenced anywhere in the document. // The basic format is: // // [1]: http://www.google.com/ "Google" // [2]: http://www.github.com/ "Github" // // Anywhere in the document, the reference can be linked by referring to its // label, i.e., 1 and 2 in this example, as in: // // This library is hosted on [Github][2], a git hosting site. // // Actual footnotes as specified in Pandoc and supported by some other Markdown // libraries such as php-markdown are also taken care of. They look like this: // // This sentence needs a bit of further explanation.[^note] // // [^note]: This is the explanation. // // Footnotes should be placed at the end of the document in an ordered list. // Inline footnotes such as: // // Inline footnotes^[Not supported.] also exist. // // are not yet supported. // reference holds all information necessary for a reference-style links or // footnotes. // // Consider this markdown with reference-style links: // // [link][ref] // // [ref]: /url/ "tooltip title" // // It will be ultimately converted to this HTML: // //

link

// // And a reference structure will be populated as follows: // // p.refs["ref"] = &reference{ // link: "/url/", // title: "tooltip title", // } // // Alternatively, reference can contain information about a footnote. Consider // this markdown: // // Text needing a footnote.[^a] // // [^a]: This is the note // // A reference structure will be populated as follows: // // p.refs["a"] = &reference{ // link: "a", // title: "This is the note", // noteID: , // } // // TODO: As you can see, it begs for splitting into two dedicated structures // for refs and for footnotes. type reference struct { link []byte title []byte noteID int // 0 if not a footnote ref hasBlock bool footnote ast.Node // a link to the Item node within a list of footnotes text []byte // only gets populated by refOverride feature with Reference.Text } func (r *reference) String() string { return fmt.Sprintf("{link: %q, title: %q, text: %q, noteID: %d, hasBlock: %v}", r.link, r.title, r.text, r.noteID, r.hasBlock) } // Check whether or not data starts with a reference link. // If so, it is parsed and stored in the list of references // (in the render struct). // Returns the number of bytes to skip to move past it, // or zero if the first line is not a reference. func isReference(p *Parser, data []byte, tabSize int) int { // up to 3 optional leading spaces if len(data) < 4 { return 0 } i := 0 for i < 3 && data[i] == ' ' { i++ } noteID := 0 // id part: anything but a newline between brackets if data[i] != '[' { return 0 } i++ if p.extensions&Footnotes != 0 { if i < len(data) && data[i] == '^' { // we can set it to anything here because the proper noteIds will // be assigned later during the second pass. It just has to be != 0 noteID = 1 i++ } } idOffset := i for i < len(data) && data[i] != '\n' && data[i] != '\r' && data[i] != ']' { i++ } if i >= len(data) || data[i] != ']' { return 0 } idEnd := i // footnotes can have empty ID, like this: [^], but a reference can not be // empty like this: []. Break early if it's not a footnote and there's no ID if noteID == 0 && idOffset == idEnd { return 0 } // spacer: colon (space | tab)* newline? (space | tab)* i++ if i >= len(data) || data[i] != ':' { return 0 } i++ for i < len(data) && (data[i] == ' ' || data[i] == '\t') { i++ } if i < len(data) && (data[i] == '\n' || data[i] == '\r') { i++ if i < len(data) && data[i] == '\n' && data[i-1] == '\r' { i++ } } for i < len(data) && (data[i] == ' ' || data[i] == '\t') { i++ } if i >= len(data) { return 0 } var ( linkOffset, linkEnd int titleOffset, titleEnd int lineEnd int raw []byte hasBlock bool ) if p.extensions&Footnotes != 0 && noteID != 0 { linkOffset, linkEnd, raw, hasBlock = scanFootnote(p, data, i, tabSize) lineEnd = linkEnd } else { linkOffset, linkEnd, titleOffset, titleEnd, lineEnd = scanLinkRef(p, data, i) } if lineEnd == 0 { return 0 } // a valid ref has been found ref := &reference{ noteID: noteID, hasBlock: hasBlock, } if noteID > 0 { // reusing the link field for the id since footnotes don't have links ref.link = data[idOffset:idEnd] // if footnote, it's not really a title, it's the contained text ref.title = raw } else { ref.link = data[linkOffset:linkEnd] ref.title = data[titleOffset:titleEnd] } // id matches are case-insensitive id := string(bytes.ToLower(data[idOffset:idEnd])) p.refs[id] = ref return lineEnd } func scanLinkRef(p *Parser, data []byte, i int) (linkOffset, linkEnd, titleOffset, titleEnd, lineEnd int) { // link: whitespace-free sequence, optionally between angle brackets if data[i] == '<' { i++ } linkOffset = i for i < len(data) && data[i] != ' ' && data[i] != '\t' && data[i] != '\n' && data[i] != '\r' { i++ } linkEnd = i if linkEnd < len(data) && data[linkOffset] == '<' && data[linkEnd-1] == '>' { linkOffset++ linkEnd-- } // optional spacer: (space | tab)* (newline | '\'' | '"' | '(' ) for i < len(data) && (data[i] == ' ' || data[i] == '\t') { i++ } if i < len(data) && data[i] != '\n' && data[i] != '\r' && data[i] != '\'' && data[i] != '"' && data[i] != '(' { return } // compute end-of-line if i >= len(data) || data[i] == '\r' || data[i] == '\n' { lineEnd = i } if i+1 < len(data) && data[i] == '\r' && data[i+1] == '\n' { lineEnd++ } // optional (space|tab)* spacer after a newline if lineEnd > 0 { i = lineEnd + 1 for i < len(data) && (data[i] == ' ' || data[i] == '\t') { i++ } } // optional title: any non-newline sequence enclosed in '"() alone on its line if i+1 < len(data) && (data[i] == '\'' || data[i] == '"' || data[i] == '(') { i++ titleOffset = i // look for EOL for i < len(data) && data[i] != '\n' && data[i] != '\r' { i++ } if i+1 < len(data) && data[i] == '\n' && data[i+1] == '\r' { titleEnd = i + 1 } else { titleEnd = i } // step back i-- for i > titleOffset && (data[i] == ' ' || data[i] == '\t') { i-- } if i > titleOffset && (data[i] == '\'' || data[i] == '"' || data[i] == ')') { lineEnd = titleEnd titleEnd = i } } return } // The first bit of this logic is the same as Parser.listItem, but the rest // is much simpler. This function simply finds the entire block and shifts it // over by one tab if it is indeed a block (just returns the line if it's not). // blockEnd is the end of the section in the input buffer, and contents is the // extracted text that was shifted over one tab. It will need to be rendered at // the end of the document. func scanFootnote(p *Parser, data []byte, i, indentSize int) (blockStart, blockEnd int, contents []byte, hasBlock bool) { if i == 0 || len(data) == 0 { return } // skip leading whitespace on first line for i < len(data) && data[i] == ' ' { i++ } blockStart = i // find the end of the line blockEnd = i for i < len(data) && data[i-1] != '\n' { i++ } // get working buffer var raw bytes.Buffer // put the first line into the working buffer raw.Write(data[blockEnd:i]) blockEnd = i // process the following lines containsBlankLine := false gatherLines: for blockEnd < len(data) { i++ // find the end of this line for i < len(data) && data[i-1] != '\n' { i++ } // if it is an empty line, guess that it is part of this item // and move on to the next line if p.isEmpty(data[blockEnd:i]) > 0 { containsBlankLine = true blockEnd = i continue } n := 0 if n = isIndented(data[blockEnd:i], indentSize); n == 0 { // this is the end of the block. // we don't want to include this last line in the index. break gatherLines } // if there were blank lines before this one, insert a new one now if containsBlankLine { raw.WriteByte('\n') containsBlankLine = false } // get rid of that first tab, write to buffer raw.Write(data[blockEnd+n : i]) hasBlock = true blockEnd = i } if data[blockEnd-1] != '\n' { raw.WriteByte('\n') } contents = raw.Bytes() return } // IsPunctuation returns true if c is a punctuation symbol. func IsPunctuation(c byte) bool { for _, r := range []byte("!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~") { if c == r { return true } } return false } // IsSpace returns true if c is a white-space charactr func IsSpace(c byte) bool { return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v' } // IsLetter returns true if c is ascii letter func IsLetter(c byte) bool { return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') } // IsAlnum returns true if c is a digit or letter // TODO: check when this is looking for ASCII alnum and when it should use unicode func IsAlnum(c byte) bool { return (c >= '0' && c <= '9') || IsLetter(c) } var URIs = [][]byte{ []byte("http://"), []byte("https://"), []byte("ftp://"), []byte("mailto:"), } var Paths = [][]byte{ []byte("/"), []byte("./"), []byte("../"), } // IsSafeURL returns true if url starts with one of the valid schemes or is a relative path. func IsSafeURL(url []byte) bool { nLink := len(url) for _, path := range Paths { nPath := len(path) linkPrefix := url[:nPath] if nLink >= nPath && bytes.Equal(linkPrefix, path) { if nLink == nPath { return true } else if IsAlnum(url[nPath]) { return true } } } for _, prefix := range URIs { // TODO: handle unicode here // case-insensitive prefix test nPrefix := len(prefix) if nLink > nPrefix { linkPrefix := bytes.ToLower(url[:nPrefix]) if bytes.Equal(linkPrefix, prefix) && IsAlnum(url[nPrefix]) { return true } } } return false } // TODO: this is not used // Replace tab characters with spaces, aligning to the next TAB_SIZE column. // always ends output with a newline /* func expandTabs(out *bytes.Buffer, line []byte, tabSize int) { // first, check for common cases: no tabs, or only tabs at beginning of line i, prefix := 0, 0 slowcase := false for i = 0; i < len(line); i++ { if line[i] == '\t' { if prefix == i { prefix++ } else { slowcase = true break } } } // no need to decode runes if all tabs are at the beginning of the line if !slowcase { for i = 0; i < prefix*tabSize; i++ { out.WriteByte(' ') } out.Write(line[prefix:]) return } // the slow case: we need to count runes to figure out how // many spaces to insert for each tab column := 0 i = 0 for i < len(line) { start := i for i < len(line) && line[i] != '\t' { _, size := utf8.DecodeRune(line[i:]) i += size column++ } if i > start { out.Write(line[start:i]) } if i >= len(line) { break } for { out.WriteByte(' ') column++ if column%tabSize == 0 { break } } i++ } } */ // Find if a line counts as indented or not. // Returns number of characters the indent is (0 = not indented). func isIndented(data []byte, indentSize int) int { if len(data) == 0 { return 0 } if data[0] == '\t' { return 1 } if len(data) < indentSize { return 0 } for i := 0; i < indentSize; i++ { if data[i] != ' ' { return 0 } } return indentSize } // Create a url-safe slug for fragments func slugify(in []byte) []byte { if len(in) == 0 { return in } out := make([]byte, 0, len(in)) sym := false for _, ch := range in { if IsAlnum(ch) { sym = false out = append(out, ch) } else if sym { continue } else { out = append(out, '-') sym = true } } var a, b int var ch byte for a, ch = range out { if ch != '-' { break } } for b = len(out) - 1; b > 0; b-- { if out[b] != '-' { break } } return out[a : b+1] } func isListItem(d ast.Node) bool { _, ok := d.(*ast.ListItem) return ok }