From 8c12c6939aab9106db14ec2d11d983bc5b29fb2c Mon Sep 17 00:00:00 2001 From: Niall Sheridan Date: Sun, 7 Jul 2019 21:33:44 +0100 Subject: Switch to modules --- vendor/github.com/homemade/scl/tokeniser.go | 279 ---------------------------- 1 file changed, 279 deletions(-) delete mode 100644 vendor/github.com/homemade/scl/tokeniser.go (limited to 'vendor/github.com/homemade/scl/tokeniser.go') diff --git a/vendor/github.com/homemade/scl/tokeniser.go b/vendor/github.com/homemade/scl/tokeniser.go deleted file mode 100644 index 0986ef8..0000000 --- a/vendor/github.com/homemade/scl/tokeniser.go +++ /dev/null @@ -1,279 +0,0 @@ -package scl - -import ( - "fmt" - "regexp" - "strings" - "unicode" -) - -var hashCommentMatcher = regexp.MustCompile(`#.+$`) -var functionMatcher = regexp.MustCompile(`^([a-zA-Z0-9_]+)\s?\((.*)\):?$`) -var shortFunctionMatcher = regexp.MustCompile(`^([a-zA-Z0-9_]+):$`) -var variableMatcher = regexp.MustCompile(`^\$([a-zA-Z_][a-zA-Z0-9_]*)$`) -var assignmentMatcher = regexp.MustCompile(`^\$([a-zA-Z_][a-zA-Z0-9_]*)\s*=\s*((.|\n)+)$`) -var declarationMatcher = regexp.MustCompile(`^\$([a-zA-Z_][a-zA-Z0-9_]*)\s*:=\s*(.+)$`) -var conditionalVariableMatcher = regexp.MustCompile(`^\$([a-zA-Z_0-9]+)\s*\?=\s*(.+)$`) -var docblockStartMatcher = regexp.MustCompile(`^/\*$`) -var docblockEndMatcher = regexp.MustCompile(`^\*\/$`) -var heredocMatcher = regexp.MustCompile(`<<([a-zA-Z]+)\s*$`) - -type tokeniser struct { - accruedComment []string -} - -func newTokeniser() *tokeniser { - return &tokeniser{} -} - -func (t *tokeniser) resetComment() { - t.accruedComment = make([]string, 0) -} - -func (t *tokeniser) stripComments(l *scannerLine) string { - - lastQuote := []rune{rune(0)} - slash := rune(47) - slashCount := 0 - - result := func() (result []byte) { - - for i, v := range []byte(l.content) { - - c := rune(v) - - switch { - case c == lastQuote[0]: - lastQuote = lastQuote[1:] - slashCount = 0 - - case unicode.In(c, unicode.Quotation_Mark): - lastQuote = append([]rune{c}, lastQuote...) - slashCount = 0 - - case c == slash && lastQuote[0] == rune(0): - - slashCount++ - if slashCount == 2 { - return result[0:(i - 1)] - } - - default: - slashCount = 0 - } - - result = append(result, v) - } - - return - }() - - return strings.Trim(string(result), " ") -} - -func (t *tokeniser) tokenise(l *scannerLine) (tokens []token, err error) { - - // Remove comments - content := t.stripComments(l) - - // If the string is empty, the entire line was a comment - if content == "" { - return []token{ - token{ - kind: tokenLineComment, - content: strings.TrimLeft(string(l.content), "/ "), - line: l, - }, - }, nil - } - - if docblockStartMatcher.MatchString(content) { - return t.tokeniseCommentStart(l, lineContent(content)) - } - - if docblockEndMatcher.MatchString(content) { - return t.tokeniseCommentEnd(l, lineContent(content)) - } - - // Mixin declarations start with a @ - if content[0] == '@' { - return t.tokeniseMixinDeclaration(l, lineContent(content)) - } - - if shortFunctionMatcher.MatchString(content) { - return t.tokeniseShortFunctionCall(l, lineContent(content)) - } - - if functionMatcher.MatchString(content) { - return t.tokeniseFunctionCall(l, lineContent(content)) - } - - if assignmentMatcher.MatchString(content) { - return t.tokeniseVariableAssignment(l, lineContent(content)) - } - - if declarationMatcher.MatchString(content) { - return t.tokeniseVariableDeclaration(l, lineContent(content)) - } - - if conditionalVariableMatcher.MatchString(content) { - return t.tokeniseConditionalVariableAssignment(l, lineContent(content)) - } - - // Assume the result is a literal - return []token{ - token{kind: tokenLiteral, content: content, line: l}, - }, nil -} - -func (t *tokeniser) tokeniseCommentStart(l *scannerLine, content lineContent) (tokens []token, err error) { - tokens = append(tokens, token{kind: tokenCommentStart, line: l}) - return -} - -func (t *tokeniser) tokeniseCommentEnd(l *scannerLine, content lineContent) (tokens []token, err error) { - tokens = append(tokens, token{kind: tokenCommentEnd, line: l}) - return -} - -func (t *tokeniser) tokeniseFunction(l *scannerLine, input string) (name string, tokens []token, err error) { - - parts := functionMatcher.FindStringSubmatch(input) - - if len(parts) < 2 { - return "", tokens, fmt.Errorf("Can't parse function signature") - } - - name = parts[1] - - if len(parts) == 3 && parts[2] != "" { - - lastQuote := rune(0) - comma := rune(0x2c) - leftBracket := rune(0x5b) - rightBracket := rune(0x5d) - - f := func(c rune) bool { - - switch { - case c == lastQuote: - lastQuote = rune(0) - return false - case lastQuote != rune(0): - return false - case unicode.In(c, unicode.Quotation_Mark): - lastQuote = c - return false - case c == leftBracket: - lastQuote = rightBracket - return false - case c == comma: - return true - default: - return false - - } - } - - arguments := strings.FieldsFunc(parts[2], f) - - for _, arg := range arguments { - - arg = strings.Trim(arg, " \t") - - if matches := variableMatcher.FindStringSubmatch(arg); len(matches) > 1 { - tokens = append(tokens, token{kind: tokenVariable, content: matches[1], line: l}) - } else if matches := assignmentMatcher.FindStringSubmatch(arg); len(matches) > 1 { - tokens = append(tokens, token{kind: tokenVariableAssignment, content: matches[1], line: l}) - tokens = append(tokens, token{kind: tokenLiteral, content: matches[2], line: l}) - } else { - tokens = append(tokens, token{kind: tokenLiteral, content: arg, line: l}) - } - } - } - - return -} - -func (t *tokeniser) tokeniseMixinDeclaration(l *scannerLine, content lineContent) (tokens []token, err error) { - - name, fntokens, fnerr := t.tokeniseFunction(l, string(content)[1:]) - - if fnerr != nil { - return tokens, fmt.Errorf("%s: %s", l, fnerr) - } - - tokens = append(tokens, token{kind: tokenMixinDeclaration, content: name, line: l}) - tokens = append(tokens, fntokens...) - - return -} - -func (t *tokeniser) tokeniseFunctionCall(l *scannerLine, content lineContent) (tokens []token, err error) { - - name, fntokens, fnerr := t.tokeniseFunction(l, string(content)) - - if fnerr != nil { - return tokens, fmt.Errorf("%s: %s", l, fnerr) - } - - tokens = append(tokens, token{kind: tokenFunctionCall, content: name, line: l}) - tokens = append(tokens, fntokens...) - - return -} - -func (t *tokeniser) tokeniseShortFunctionCall(l *scannerLine, content lineContent) (tokens []token, err error) { - - parts := shortFunctionMatcher.FindStringSubmatch(string(content)) - - if len(parts) > 0 { - return []token{ - token{kind: tokenFunctionCall, content: parts[1], line: l}, - }, nil - } - - return tokens, fmt.Errorf("Failed to parse short function call") -} - -func (t *tokeniser) tokeniseVariableAssignment(l *scannerLine, content lineContent) (tokens []token, err error) { - - parts := assignmentMatcher.FindStringSubmatch(string(content)) - - if len(parts) > 0 { - return []token{ - token{kind: tokenVariableAssignment, content: parts[1], line: l}, - token{kind: tokenLiteral, content: parts[2], line: l}, - }, nil - } - - return tokens, fmt.Errorf("Failed to parse variable assignment") -} - -func (t *tokeniser) tokeniseVariableDeclaration(l *scannerLine, content lineContent) (tokens []token, err error) { - - parts := declarationMatcher.FindStringSubmatch(string(content)) - - if len(parts) > 0 { - return []token{ - token{kind: tokenVariableDeclaration, content: parts[1], line: l}, - token{kind: tokenLiteral, content: parts[2], line: l}, - }, nil - } - - return tokens, fmt.Errorf("Failed to parse variable declaration") -} - -func (t *tokeniser) tokeniseConditionalVariableAssignment(l *scannerLine, content lineContent) (tokens []token, err error) { - - parts := conditionalVariableMatcher.FindStringSubmatch(string(content)) - - if len(parts) > 0 { - return []token{ - token{kind: tokenConditionalVariableAssignment, content: parts[1], line: l}, - token{kind: tokenLiteral, content: parts[2], line: l}, - }, nil - } - - return tokens, fmt.Errorf("Failed to parse conditional variable assignment") -} -- cgit v1.2.3