Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files. =================================================================== RCS file: /ftp/cvs/cvsroot/pkgsrc/pkgtools/pkglint/files/Attic/shell_test.go,v rcsdiff: /ftp/cvs/cvsroot/pkgsrc/pkgtools/pkglint/files/Attic/shell_test.go,v: warning: Unknown phrases like `commitid ...;' are present. retrieving revision 1.55 retrieving revision 1.56 diff -u -p -r1.55 -r1.56 --- pkgsrc/pkgtools/pkglint/files/Attic/shell_test.go 2019/11/04 18:44:21 1.55 +++ pkgsrc/pkgtools/pkglint/files/Attic/shell_test.go 2019/11/17 01:26:25 1.56 @@ -5,154 +5,171 @@ import ( "strings" ) -func (s *Suite) Test_splitIntoShellTokens__line_continuation(c *check.C) { +func (s *Suite) Test_SimpleCommandChecker_handleForbiddenCommand(c *check.C) { t := s.Init(c) - words, rest := splitIntoShellTokens(dummyLine, "if true; then \\") + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + "\t${RUN} mktexlsr; texconfig") - t.CheckDeepEquals(words, []string{"if", "true", ";", "then"}) - t.CheckEquals(rest, "\\") + mklines.Check() t.CheckOutputLines( - "WARN: Internal pkglint error in ShTokenizer.ShAtom at \"\\\\\" (quoting=plain).") + "ERROR: Makefile:3: \"mktexlsr\" must not be used in Makefiles.", + "ERROR: Makefile:3: \"texconfig\" must not be used in Makefiles.") } -func (s *Suite) Test_splitIntoShellTokens__dollar_slash(c *check.C) { +func (s *Suite) Test_SimpleCommandChecker_handleCommandVariable(c *check.C) { t := s.Init(c) - words, rest := splitIntoShellTokens(dummyLine, "pax -s /.*~$$//g") - - t.CheckDeepEquals(words, []string{"pax", "-s", "/.*~$$//g"}) - t.CheckEquals(rest, "") -} - -func (s *Suite) Test_splitIntoShellTokens__dollar_subshell(c *check.C) { - t := s.Init(c) + t.SetUpTool("perl", "PERL5", AtRunTime) + t.SetUpTool("perl6", "PERL6", Nowhere) + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + "PERL5_VARS_CMD=\t${PERL5:Q}", + "PERL5_VARS_CMD=\t${PERL6:Q}", + "", + "pre-configure:", + "\t${PERL5_VARS_CMD} -e 'print 12345'") - words, rest := splitIntoShellTokens(dummyLine, "id=$$(${AWK} '{print}' < ${WRKSRC}/idfile) && echo \"$$id\"") + mklines.Check() - t.CheckDeepEquals(words, []string{"id=$$(${AWK} '{print}' < ${WRKSRC}/idfile)", "&&", "echo", "\"$$id\""}) - t.CheckEquals(rest, "") + // FIXME: In PERL5:Q and PERL6:Q, the :Q is wrong. + t.CheckOutputLines( + "WARN: Makefile:4: The \"${PERL6:Q}\" tool is used but not added to USE_TOOLS.") } -func (s *Suite) Test_splitIntoShellTokens__semicolons(c *check.C) { +func (s *Suite) Test_SimpleCommandChecker_handleCommandVariable__parameterized(c *check.C) { t := s.Init(c) - words, rest := splitIntoShellTokens(dummyLine, "word1 word2;;;") - - t.CheckDeepEquals(words, []string{"word1", "word2", ";;", ";"}) - t.CheckEquals(rest, "") -} + t.SetUpPackage("category/package") + G.Pkg = NewPackage(t.File("category/package")) + t.FinishSetUp() -func (s *Suite) Test_splitIntoShellTokens__whitespace(c *check.C) { - t := s.Init(c) + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + "MY_TOOL.i386=\t${PREFIX}/bin/tool-i386", + "MY_TOOL.x86_64=\t${PREFIX}/bin/tool-x86_64", + "", + "pre-configure:", + "\t${MY_TOOL.amd64} -e 'print 12345'", + "\t${UNKNOWN_TOOL}") - text := "\t${RUN} cd ${WRKSRC}&&(${ECHO} ${PERL5:Q};${ECHO})|${BASH} ./install" - words, rest := splitIntoShellTokens(dummyLine, text) + mklines.Check() - t.CheckDeepEquals(words, []string{ - "${RUN}", - "cd", "${WRKSRC}", - "&&", "(", "${ECHO}", "${PERL5:Q}", ";", "${ECHO}", ")", - "|", "${BASH}", "./install"}) - t.CheckEquals(rest, "") + t.CheckOutputLines( + "WARN: Makefile:8: Unknown shell command \"${UNKNOWN_TOOL}\".", + "WARN: Makefile:8: UNKNOWN_TOOL is used but not defined.") } -func (s *Suite) Test_splitIntoShellTokens__finished_dquot(c *check.C) { +func (s *Suite) Test_SimpleCommandChecker_handleCommandVariable__followed_by_literal(c *check.C) { t := s.Init(c) - text := "\"\"" - words, rest := splitIntoShellTokens(dummyLine, text) - - t.CheckDeepEquals(words, []string{"\"\""}) - t.CheckEquals(rest, "") -} + t.SetUpPackage("category/package") + G.Pkg = NewPackage(t.File("category/package")) + t.FinishSetUp() -func (s *Suite) Test_splitIntoShellTokens__unfinished_dquot(c *check.C) { - t := s.Init(c) + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + "QTDIR=\t${PREFIX}", + "", + "pre-configure:", + "\t${QTDIR}/bin/release") - text := "\t\"" - words, rest := splitIntoShellTokens(dummyLine, text) + mklines.Check() - c.Check(words, check.IsNil) - t.CheckEquals(rest, "\"") + t.CheckOutputEmpty() } -func (s *Suite) Test_splitIntoShellTokens__unescaped_dollar_in_dquot(c *check.C) { +// The package Makefile and other .mk files in a package directory +// may use any shell commands defined by any included files. +// But only if the package is checked as a whole. +// +// On the contrary, when pkglint checks a single .mk file, these +// commands are not guaranteed to be defined, not even when the +// .mk file includes the file defining the command. +// FIXME: This paragraph sounds wrong. All commands from included files should be valid. +// +// The PYTHON_BIN variable below must not be called *_CMD, or another code path is taken. +func (s *Suite) Test_SimpleCommandChecker_handleCommandVariable__from_package(c *check.C) { t := s.Init(c) - text := "echo \"$$\"" - words, rest := splitIntoShellTokens(dummyLine, text) + pkg := t.SetUpPackage("category/package", + "post-install:", + "\t${PYTHON_BIN}", + "", + ".include \"extra.mk\"") + t.CreateFileLines("category/package/extra.mk", + MkCvsID, + "PYTHON_BIN=\tmy_cmd") + t.FinishSetUp() - t.CheckDeepEquals(words, []string{"echo", "\"$$\""}) - t.CheckEquals(rest, "") + G.Check(pkg) t.CheckOutputEmpty() } -func (s *Suite) Test_splitIntoShellTokens__varuse_with_embedded_space_and_other_vars(c *check.C) { +func (s *Suite) Test_SimpleCommandChecker_checkRegexReplace(c *check.C) { t := s.Init(c) - varuseWord := "${GCONF_SCHEMAS:@.s.@${INSTALL_DATA} ${WRKSRC}/src/common/dbus/${.s.} ${DESTDIR}${GCONF_SCHEMAS_DIR}/@}" - words, rest := splitIntoShellTokens(dummyLine, varuseWord) + test := func(cmd string, diagnostics ...string) { + t.SetUpTool("pax", "PAX", AtRunTime) + t.SetUpTool("sed", "SED", AtRunTime) + mklines := t.NewMkLines("Makefile", + MkCvsID, + "pre-configure:", + "\t"+cmd) - t.CheckDeepEquals(words, []string{varuseWord}) - t.CheckEquals(rest, "") -} + mklines.Check() -// Two shell variables, next to each other, -// are two separate atoms but count as a single token. -func (s *Suite) Test_splitIntoShellTokens__two_shell_variables(c *check.C) { - t := s.Init(c) + t.CheckOutput(diagnostics) + } - code := "echo $$i$$j" - words, rest := splitIntoShellTokens(dummyLine, code) + test("${PAX} -s s,.*,, src dst", + "WARN: Makefile:3: Substitution commands like \"s,.*,,\" should always be quoted.") - t.CheckDeepEquals(words, []string{"echo", "$$i$$j"}) - t.CheckEquals(rest, "") -} + test("pax -s s,.*,, src dst", + "WARN: Makefile:3: Substitution commands like \"s,.*,,\" should always be quoted.") -func (s *Suite) Test_splitIntoShellTokens__varuse_with_embedded_space(c *check.C) { - t := s.Init(c) + test("${SED} -e s,.*,, src dst", + "WARN: Makefile:3: Substitution commands like \"s,.*,,\" should always be quoted.") - words, rest := splitIntoShellTokens(dummyLine, "${VAR:S/ /_/g}") + test("sed -e s,.*,, src dst", + "WARN: Makefile:3: Substitution commands like \"s,.*,,\" should always be quoted.") - t.CheckDeepEquals(words, []string{"${VAR:S/ /_/g}"}) - t.CheckEquals(rest, "") -} + // The * is properly enclosed in quotes. + test("sed -e 's,.*,,' -e \"s,-*,,\"", + nil...) -func (s *Suite) Test_splitIntoShellTokens__redirect(c *check.C) { - t := s.Init(c) + // The * is properly escaped. + test("sed -e s,.\\*,,", + nil...) - words, rest := splitIntoShellTokens(dummyLine, "echo 1>output 2>>append 3>|clobber 4>&5 6>append") + test("pax -s s,\\.orig,, src dst", + nil...) - t.CheckDeepEquals(words, []string{ - "echo", - "1>", "output", - "2>>", "append", - "3>|", "clobber", - "4>&", "5", - "6<", "input", - ">>", "append"}) - t.CheckEquals(rest, "") + test("sed -e s,a,b,g src dst", + nil...) - words, rest = splitIntoShellTokens(dummyLine, "echo 1> output 2>> append 3>| clobber 4>& 5 6< input >> append") + // TODO: Merge the code with BtSedCommands. - t.CheckDeepEquals(words, []string{ - "echo", - "1>", "output", - "2>>", "append", - "3>|", "clobber", - "4>&", "5", - "6<", "input", - ">>", "append"}) - t.CheckEquals(rest, "") + // TODO: Finally, remove the G.Testing from the main code. + // Then, remove this test case. + G.Testing = false + test("sed -e s,.*,match,", + nil...) + G.Testing = true } -func (s *Suite) Test_ShellLineChecker_CheckShellCommandLine(c *check.C) { +func (s *Suite) Test_SimpleCommandChecker_checkAutoMkdirs(c *check.C) { t := s.Init(c) t.SetUpVartypes() + // TODO: Check whether these tools are actually necessary for this test. t.SetUpTool("awk", "AWK", AtRunTime) t.SetUpTool("cp", "CP", AtRunTime) t.SetUpTool("echo", "", AtRunTime) @@ -171,198 +188,393 @@ func (s *Suite) Test_ShellLineChecker_Ch t.CheckOutput(diagnostics) } - test("@# Comment", + // AUTO_MKDIRS applies only when installing directories. + test("${RUN} ${INSTALL} -c ${WRKSRC}/file ${PREFIX}/bin/", nil...) - test("uname=`uname`; echo $$uname; echo; ${PREFIX}/bin/command", - "WARN: filename.mk:1: Unknown shell command \"uname\".", - "WARN: filename.mk:1: Please switch to \"set -e\" mode "+ - "before using a semicolon (after \"uname=`uname`\") to separate commands.") + // TODO: Warn that ${INSTALL} -d can only handle a single directory. + test("${RUN} ${INSTALL} -m 0755 -d ${PREFIX}/first ${PREFIX}/second", + "NOTE: filename.mk:1: You can use \"INSTALLATION_DIRS+= first\" instead of \"${INSTALL} -d\".", + "NOTE: filename.mk:1: You can use \"INSTALLATION_DIRS+= second\" instead of \"${INSTALL} -d\".") - test("echo ${PKGNAME:Q}", // VucQuotPlain - "NOTE: filename.mk:1: The :Q modifier isn't necessary for ${PKGNAME} here.") + G.Pkg = NewPackage(t.File("category/pkgbase")) + G.Pkg.Plist.Dirs["share/pkgbase"] = &PlistLine{ + t.NewLine("PLIST", 123, "share/pkgbase/file"), + nil, + "share/pkgbase/file"} - test("echo \"${CFLAGS:Q}\"", // VucQuotDquot + // A directory that is found in the PLIST. + // TODO: Add a test for using this command inside a conditional; + // the note should not appear then. + test("${RUN} ${INSTALL_DATA_DIR} share/pkgbase ${PREFIX}/share/pkgbase", + "NOTE: filename.mk:1: You can use AUTO_MKDIRS=yes or \"INSTALLATION_DIRS+= share/pkgbase\" "+ + "instead of \"${INSTALL_DATA_DIR}\".", + "WARN: filename.mk:1: The INSTALL_*_DIR commands can only handle one directory at a time.") - // ShellLineChecker.checkVaruseToken - "WARN: filename.mk:1: The :Q modifier should not be used inside quotes.", + // Directories from .for loops are too dynamic to be replaced with AUTO_MKDIRS. + // TODO: Expand simple .for loops. + test("${RUN} ${INSTALL_DATA_DIR} ${PREFIX}/${dir}", + "WARN: filename.mk:1: dir is used but not defined.") - // ShellLineChecker.checkVaruseToken - // MkLineChecker.CheckVaruse - // MkLineChecker.checkVarUseQuoting - "WARN: filename.mk:1: Please use ${CFLAGS:M*:Q} instead of ${CFLAGS:Q} "+ - "and make sure the variable appears outside of any quoting characters.") + // A directory that is not found in the PLIST would not be created by AUTO_MKDIRS, + // therefore only INSTALLATION_DIRS is suggested. + test("${RUN} ${INSTALL_DATA_DIR} ${PREFIX}/share/other", + "NOTE: filename.mk:1: You can use \"INSTALLATION_DIRS+= share/other\" instead of \"${INSTALL_DATA_DIR}\".") +} - test("echo '${COMMENT:Q}'", // VucQuotSquot - "WARN: filename.mk:1: The :Q modifier should not be used inside quotes.", - "WARN: filename.mk:1: Please move ${COMMENT:Q} outside of any quoting characters.") +func (s *Suite) Test_SimpleCommandChecker_checkAutoMkdirs__redundant(c *check.C) { + t := s.Init(c) - test("echo target=$@ exitcode=$$? '$$' \"\\$$\"", - "WARN: filename.mk:1: Please use \"${.TARGET}\" instead of \"$@\".", - "WARN: filename.mk:1: The $? shell variable is often not available in \"set -e\" mode.") + t.SetUpPackage("category/package", + "AUTO_MKDIRS=\t\tyes", + "INSTALLATION_DIRS+=\tshare/redundant", + "INSTALLATION_DIRS+=\tnot/redundant ${EGDIR}") + t.CreateFileLines("category/package/PLIST", + PlistCvsID, + "share/redundant/file", + "${EGDIR}/file") - test("echo $$@", - "WARN: filename.mk:1: The $@ shell variable should only be used in double quotes.") + t.Main("-Wall", "-q", "category/package") - // No warning about a possibly missed variable name. - // This occurs only rarely, and typically as part of a regular expression - // where it is used intentionally. - test("echo \"$$\"", // As seen by make(1); the shell sees: echo "$" - nil...) + t.CheckOutputLines( + "NOTE: ~/category/package/Makefile:21: The directory \"share/redundant\" "+ + "is redundant in INSTALLATION_DIRS.", + // The below is not proven to be always correct. It assumes that a + // variable in the Makefile has the same value as the corresponding + // variable from PLIST_SUBST. Violating this assumption would be + // confusing to the pkgsrc developers, therefore it's a safe bet. + // A notable counterexample is PKGNAME in PLIST, which corresponds + // to PKGNAME_NOREV in the package Makefile. + "NOTE: ~/category/package/Makefile:22: The directory \"${EGDIR}\" "+ + "is redundant in INSTALLATION_DIRS.") +} - test("echo \"\\n\"", - nil...) +// The AUTO_MKDIRS code in mk/install/install.mk (install-dirs-from-PLIST) +// skips conditional directories, as well as directories with placeholders. +func (s *Suite) Test_SimpleCommandChecker_checkAutoMkdirs__conditional_PLIST(c *check.C) { + t := s.Init(c) - test("${RUN} for f in *.c; do echo $${f%.c}; done", - nil...) + t.SetUpPackage("category/package", + "LIB_SUBDIR=\tsubdir", + "", + "do-install:", + "\t${RUN} ${INSTALL_DATA_DIR} ${PREFIX}/libexec/always", + "\t${RUN} ${INSTALL_DATA_DIR} ${PREFIX}/libexec/conditional", + "\t${RUN} ${INSTALL_DATA_DIR} ${PREFIX}/${LIB_SUBDIR}", + ) + t.Chdir("category/package") + t.CreateFileLines("PLIST", + PlistCvsID, + "libexec/always/always", + "${LIB_SUBDIR}/file", + "${PLIST.cond}libexec/conditional/conditional") + t.FinishSetUp() - test("${RUN} set +x; echo $${variable+set}", - nil...) + G.checkdirPackage(".") - // Based on mail/thunderbird/Makefile, rev. 1.159 - test("${RUN} subdir=\"`unzip -c \"$$e\" install.rdf | awk '/re/ { print \"hello\" }'`\"", - "WARN: filename.mk:1: Double quotes inside backticks inside double quotes are error prone.", - "WARN: filename.mk:1: The exitcode of \"unzip\" at the left of the | operator is ignored.") + // As libexec/conditional will not be created automatically, + // AUTO_MKDIRS must not be suggested in that line. + t.CheckOutputLines( + "NOTE: Makefile:23: You can use AUTO_MKDIRS=yes "+ + "or \"INSTALLATION_DIRS+= libexec/always\" "+ + "instead of \"${INSTALL_DATA_DIR}\".", + "NOTE: Makefile:24: You can use "+ + "\"INSTALLATION_DIRS+= libexec/conditional\" "+ + "instead of \"${INSTALL_DATA_DIR}\".", + "NOTE: Makefile:25: You can use "+ + "\"INSTALLATION_DIRS+= ${LIB_SUBDIR}\" "+ + "instead of \"${INSTALL_DATA_DIR}\".") +} - // From mail/thunderbird/Makefile, rev. 1.159 - test(""+ - "${RUN} for e in ${XPI_FILES}; do "+ - " subdir=\"`${UNZIP_CMD} -c \"$$e\" install.rdf | "+ - ""+"awk '/.../ {print;exit;}'`\" && "+ - " ${MKDIR} \"${WRKDIR}/extensions/$$subdir\" && "+ - " cd \"${WRKDIR}/extensions/$$subdir\" && "+ - " ${UNZIP_CMD} -aqo $$e; "+ - "done", - "WARN: filename.mk:1: XPI_FILES is used but not defined.", - "WARN: filename.mk:1: Double quotes inside backticks inside double quotes are error prone.", - "WARN: filename.mk:1: The exitcode of \"${UNZIP_CMD}\" at the left of the | operator is ignored.") +// This test ensures that the command line options to INSTALL_*_DIR are properly +// parsed and do not lead to "can only handle one directory at a time" warnings. +func (s *Suite) Test_SimpleCommandChecker_checkInstallMulti(c *check.C) { + t := s.Init(c) - // From x11/wxGTK28/Makefile - // TODO: Why is TOOLS_PATH.msgfmt not recognized? - // At least, the warning should be more specific, mentioning USE_TOOLS. - test(""+ - "set -e; cd ${WRKSRC}/locale; "+ - "for lang in *.po; do "+ - " [ \"$${lang}\" = \"wxstd.po\" ] && continue; "+ - " ${TOOLS_PATH.msgfmt} -c -o \"$${lang%.po}.mo\" \"$${lang}\"; "+ - "done", - "WARN: filename.mk:1: Unknown shell command \"[\".", - "WARN: filename.mk:1: Unknown shell command \"${TOOLS_PATH.msgfmt}\".") + t.SetUpVartypes() + mklines := t.NewMkLines("install.mk", + MkCvsID, + "", + "do-install:", + "\t${INSTALL_PROGRAM_DIR} -m 0555 -g ${APACHE_GROUP} -o ${APACHE_USER} \\", + "\t\t${DESTDIR}${PREFIX}/lib/apache-modules") - test("@cp from to", - "WARN: filename.mk:1: The shell command \"cp\" should not be hidden.") + mklines.Check() - test("-cp from to", - "WARN: filename.mk:1: Using a leading \"-\" to suppress errors is deprecated.") + t.CheckOutputLines( + "NOTE: install.mk:4--5: You can use \"INSTALLATION_DIRS+= lib/apache-modules\" " + + "instead of \"${INSTALL_PROGRAM_DIR}\".") +} - test("-${MKDIR} deeply/nested/subdir", - "WARN: filename.mk:1: Using a leading \"-\" to suppress errors is deprecated.") +func (s *Suite) Test_SimpleCommandChecker_checkPaxPe(c *check.C) { + t := s.Init(c) - G.Pkg = NewPackage(t.File("category/pkgbase")) - G.Pkg.Plist.Dirs["share/pkgbase"] = &PlistLine{ - t.NewLine("PLIST", 123, "share/pkgbase/file"), - nil, - "share/pkgbase/file"} + t.SetUpVartypes() + t.SetUpTool("pax", "PAX", AtRunTime) + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + "do-install:", + "\t${RUN} pax -pe ${WRKSRC} ${DESTDIR}${PREFIX}", + "\t${RUN} ${PAX} -pe ${WRKSRC} ${DESTDIR}${PREFIX}") - // A directory that is found in the PLIST. - // TODO: Add a test for using this command inside a conditional; - // the note should not appear then. - test("${RUN} ${INSTALL_DATA_DIR} share/pkgbase ${PREFIX}/share/pkgbase", - "NOTE: filename.mk:1: You can use AUTO_MKDIRS=yes or \"INSTALLATION_DIRS+= share/pkgbase\" "+ - "instead of \"${INSTALL_DATA_DIR}\".", - "WARN: filename.mk:1: The INSTALL_*_DIR commands can only handle one directory at a time.") + mklines.Check() - // A directory that is not found in the PLIST. - test("${RUN} ${INSTALL_DATA_DIR} ${PREFIX}/share/other", - "NOTE: filename.mk:1: You can use \"INSTALLATION_DIRS+= share/other\" instead of \"${INSTALL_DATA_DIR}\".") + t.CheckOutputLines( + "WARN: Makefile:4: Please use the -pp option to pax(1) instead of -pe.", + "WARN: Makefile:5: Please use the -pp option to pax(1) instead of -pe.") +} - G.Pkg = nil +func (s *Suite) Test_SimpleCommandChecker_checkEchoN(c *check.C) { + t := s.Init(c) - // See PR 46570, item "1. It does not" - // No warning about missing error checking ("set -e"). - test("for x in 1 2 3; do echo \"$$x\" || exit 1; done", - nil...) + t.SetUpTool("echo", "ECHO", AtRunTime) + t.SetUpTool("echo -n", "ECHO_N", AtRunTime) + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + "do-install:", + "\t${RUN} ${ECHO} -n 'Computing...'", + "\t${RUN} ${ECHO_N} 'Computing...'", + "\t${RUN} ${ECHO} 'Computing...'") + + mklines.Check() + + t.CheckOutputLines( + "WARN: Makefile:4: Please use ${ECHO_N} instead of \"echo -n\".") } -// TODO: Document in detail that strip is not a regular tool. -func (s *Suite) Test_ShellLineChecker_CheckShellCommandLine__strip(c *check.C) { +func (s *Suite) Test_ShellLineChecker__shell_comment_with_line_continuation(c *check.C) { t := s.Init(c) - test := func(shellCommand string) { - mklines := t.NewMkLines("filename.mk", - "\t"+shellCommand) + t.SetUpTool("echo", "", AtRunTime) - mklines.ForEach(func(mkline *MkLine) { - ck := NewShellLineChecker(mklines, mkline) - ck.CheckShellCommandLine(mkline.ShellCommand()) - }) + test := func(lines ...string) { + i := 0 + for ; i < len(lines) && hasPrefix(lines[i], "\t"); i++ { + } + + mklines := t.SetUpFileMkLines("Makefile", + append([]string{MkCvsID, "pre-install:"}, + lines[:i]...)...) + + mklines.Check() + + t.CheckOutput(lines[i:]) } - test("${STRIP} executable") + // The comment can start at the beginning of a follow-up line. + test( + "\techo first; \\", + "\t# comment at the beginning of a command \\", + "\techo \"hello\"", - t.CheckOutputLines( - "WARN: filename.mk:1: Unknown shell command \"${STRIP}\".", - "WARN: filename.mk:1: STRIP is used but not defined.") + // TODO: Warn that the "echo hello" is commented out. + ) - t.SetUpVartypes() + // The comment can start at the beginning of a simple command. + test( + "\techo first; # comment at the beginning of a command \\", + "\techo \"hello\"", - test("${STRIP} executable") + // TODO: Warn that the "echo hello" is commented out. + ) - t.CheckOutputEmpty() + // The comment can start at a word in the middle of a command. + test( + // TODO: Warn that the "echo hello" is commented out. + "\techo # comment starts inside a command \\", + "\techo \"hello\"") + + // If the comment starts in the last line, there's no further + // line that might be commented out accidentally. + test( + "\techo 'first line'; \\", + "\t# comment in last line") } -func (s *Suite) Test_ShellLineChecker_CheckShellCommandLine__nofix(c *check.C) { +func (s *Suite) Test_ShellLineChecker_checkConditionalCd(c *check.C) { t := s.Init(c) - t.SetUpVartypes() - t.SetUpTool("echo", "", AtRunTime) + t.SetUpTool("ls", "", AtRunTime) + t.SetUpTool("printf", "", AtRunTime) + t.SetUpTool("wc", "", AtRunTime) mklines := t.NewMkLines("Makefile", - "\techo ${PKGNAME:Q}") - ck := NewShellLineChecker(mklines, mklines.mklines[0]) + MkCvsID, + "pre-configure:", + "\t${RUN} while cd ..; do printf .; done", + "\t${RUN} while cd .. && cd ..; do printf .; done", // Unusual, therefore no warning. + "\t${RUN} if cd ..; then printf .; fi", + "\t${RUN} ! cd ..", + "\t${RUN} if cd ..; printf 'ok\\n'; then printf .; fi", + "\t${RUN} if cd .. | wc -l; then printf .; fi", // Unusual, therefore no warning. + "\t${RUN} if cd .. && cd ..; then printf .; fi") // Unusual, therefore no warning. - ck.CheckShellCommandLine("echo ${PKGNAME:Q}") + mklines.Check() t.CheckOutputLines( - "NOTE: Makefile:1: The :Q modifier isn't necessary for ${PKGNAME} here.") + "ERROR: Makefile:3: The Solaris /bin/sh cannot handle \"cd\" inside conditionals.", + "ERROR: Makefile:5: The Solaris /bin/sh cannot handle \"cd\" inside conditionals.", + "WARN: Makefile:6: The Solaris /bin/sh does not support negation of shell commands.", + "WARN: Makefile:8: The exitcode of \"cd\" at the left of the | operator is ignored.") } -func (s *Suite) Test_ShellLineChecker_CheckShellCommandLine__show_autofix(c *check.C) { +func (s *Suite) Test_ShellLineChecker_checkSetE__simple_commands(c *check.C) { t := s.Init(c) - t.SetUpCommandLine("-Wall", "--show-autofix") - t.SetUpVartypes() t.SetUpTool("echo", "", AtRunTime) + t.SetUpTool("rm", "", AtRunTime) + t.SetUpTool("touch", "", AtRunTime) mklines := t.NewMkLines("Makefile", - "\techo ${PKGNAME:Q}") - ck := NewShellLineChecker(mklines, mklines.mklines[0]) + MkCvsID, + "pre-configure:", + "\techo 1; echo 2; echo 3", + "\techo 1; touch file; rm file", + "\techo 1; var=value; echo 3") - ck.CheckShellCommandLine("echo ${PKGNAME:Q}") + mklines.Check() t.CheckOutputLines( - "NOTE: Makefile:1: The :Q modifier isn't necessary for ${PKGNAME} here.", - "AUTOFIX: Makefile:1: Replacing \"${PKGNAME:Q}\" with \"${PKGNAME}\".") + "WARN: Makefile:4: Please switch to \"set -e\" mode before using a semicolon " + + "(after \"touch file\") to separate commands.") } -func (s *Suite) Test_ShellLineChecker_CheckShellCommandLine__autofix(c *check.C) { +func (s *Suite) Test_ShellLineChecker_checkSetE__compound_commands(c *check.C) { t := s.Init(c) - t.SetUpCommandLine("-Wall", "--autofix") - t.SetUpVartypes() t.SetUpTool("echo", "", AtRunTime) + t.SetUpTool("touch", "", AtRunTime) mklines := t.NewMkLines("Makefile", - "\techo ${PKGNAME:Q}") - ck := NewShellLineChecker(mklines, mklines.mklines[0]) + MkCvsID, + "pre-configure:", + "\ttouch file; for f in file; do echo \"$$f\"; done", + "\tfor f in file; do echo \"$$f\"; done; touch file", + "\ttouch 1; touch 2; touch 3; touch 4") - ck.CheckShellCommandLine("echo ${PKGNAME:Q}") + mklines.Check() t.CheckOutputLines( - "AUTOFIX: Makefile:1: Replacing \"${PKGNAME:Q}\" with \"${PKGNAME}\".") - - // TODO: There should be a general way of testing a code in the three modes: - // default, --show-autofix, --autofix. + "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon "+ + "(after \"touch file\") to separate commands.", + "WARN: Makefile:5: Please switch to \"set -e\" mode before using a semicolon "+ + "(after \"touch 1\") to separate commands.") +} + +func (s *Suite) Test_ShellLineChecker_checkSetE__no_tracing(c *check.C) { + t := s.Init(c) + + t.SetUpTool("touch", "", AtRunTime) + mklines := t.NewMkLines("Makefile", + MkCvsID, + "pre-configure:", + "\ttouch 1; touch 2") + t.DisableTracing() + + mklines.Check() + + t.CheckOutputLines( + "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon " + + "(after \"touch 1\") to separate commands.") +} + +func (s *Suite) Test_ShellLineChecker_canFail(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + t.SetUpTool("basename", "", AtRunTime) + t.SetUpTool("dirname", "", AtRunTime) + t.SetUpTool("echo", "", AtRunTime) + t.SetUpTool("env", "", AtRunTime) + t.SetUpTool("grep", "GREP", AtRunTime) + t.SetUpTool("sed", "", AtRunTime) + t.SetUpTool("touch", "", AtRunTime) + t.SetUpTool("tr", "tr", AtRunTime) + t.SetUpTool("true", "TRUE", AtRunTime) + + test := func(cmd string, diagnostics ...string) { + mklines := t.NewMkLines("Makefile", + MkCvsID, + "pre-configure:", + "\t"+cmd+" ; echo 'done.'") + + mklines.Check() + + t.CheckOutput(diagnostics) + } + + test("socklen=`${GREP} 'expr' ${WRKSRC}/config.h`", + "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon "+ + "(after \"socklen=`${GREP} 'expr' ${WRKSRC}/config.h`\") to separate commands.") + + test("socklen=`${GREP} 'expr' ${WRKSRC}/config.h || ${TRUE}`", + nil...) + + test("socklen=$$(expr 16)", + "WARN: Makefile:3: Invoking subshells via $(...) is not portable enough.", + "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon "+ + "(after \"socklen=$$(expr 16)\") to separate commands.") + + test("socklen=$$(expr 16 || true)", + "WARN: Makefile:3: Invoking subshells via $(...) is not portable enough.") + + test("socklen=$$(expr 16 || ${TRUE})", + "WARN: Makefile:3: Invoking subshells via $(...) is not portable enough.") + + test("${ECHO_MSG} \"Message\"", + nil...) + + test("${FAIL_MSG} \"Failure\"", + "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon "+ + "(after \"${FAIL_MSG} \\\"Failure\\\"\") to separate commands.") + + test("set -x", + "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon "+ + "(after \"set -x\") to separate commands.") + + test("echo 'input' | sed -e s,in,out,", + nil...) + + test("sed -e s,in,out,", + nil...) + + test("sed s,in,out,", + nil...) + + test("grep input", + nil...) + + test("grep pattern file...", + "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon "+ + "(after \"grep pattern file...\") to separate commands.") + + test("touch file", + "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon "+ + "(after \"touch file\") to separate commands.") + + test("echo 'starting'", + nil...) + + test("echo 'logging' > log", + "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon "+ + "(after \"echo 'logging'\") to separate commands.") + + test("echo 'to stderr' 1>&2", + nil...) + + test("echo 'hello' | tr -d 'aeiou'", + nil...) + + test("env | grep '^PATH='", + nil...) + + test("basename dir/file", + nil...) + + test("dirname dir/file", + nil...) } -func (s *Suite) Test_ShellProgramChecker_checkPipeExitcode(c *check.C) { +func (s *Suite) Test_ShellLineChecker_checkPipeExitcode(c *check.C) { t := s.Init(c) t.SetUpVartypes() @@ -401,285 +613,261 @@ func (s *Suite) Test_ShellProgramChecker "WARN: Makefile:12: The exitcode of the command at the left of the | operator is ignored.") } -// TODO: Document the exact purpose of this test, or split it into useful tests. -func (s *Suite) Test_ShellLineChecker_CheckShellCommandLine__implementation(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - mklines := t.NewMkLines("filename.mk", - "# dummy") - ck := NewShellLineChecker(mklines, mklines.mklines[0]) - - // foobar="`echo \"foo bar\"`" - text := "foobar=\"`echo \\\"foo bar\\\"`\"" - - tokens, rest := splitIntoShellTokens(dummyLine, text) - - t.CheckDeepEquals(tokens, []string{text}) - t.CheckEquals(rest, "") - - mklines.ForEach(func(mkline *MkLine) { ck.CheckWord(text, false, RunTime) }) - - t.CheckOutputLines( - "WARN: filename.mk:1: Unknown shell command \"echo\".") - - mklines.ForEach(func(mkline *MkLine) { ck.CheckShellCommandLine(text) }) - - // No parse errors - t.CheckOutputLines( - "WARN: filename.mk:1: Unknown shell command \"echo\".") -} - -func (s *Suite) Test_ShellLineChecker_CheckShellCommandLine__dollar_without_variable(c *check.C) { +func (s *Suite) Test_ShellLineChecker_CheckShellCommandLine(c *check.C) { t := s.Init(c) t.SetUpVartypes() - t.SetUpTool("pax", "", AtRunTime) - mklines := t.NewMkLines("filename.mk", - "# dummy") - ck := NewShellLineChecker(mklines, mklines.mklines[0]) - - ck.CheckShellCommandLine("pax -rwpp -s /.*~$$//g . ${DESTDIR}${PREFIX}") - - t.CheckOutputLines( - "WARN: filename.mk:1: Substitution commands like \"/.*~$$//g\" should always be quoted.") -} + t.SetUpTool("awk", "AWK", AtRunTime) + t.SetUpTool("cp", "CP", AtRunTime) + t.SetUpTool("echo", "", AtRunTime) + t.SetUpTool("mkdir", "MKDIR", AtRunTime) // This is actually "mkdir -p". + t.SetUpTool("unzip", "UNZIP_CMD", AtRunTime) -func (s *Suite) Test_ShellLineChecker_CheckWord(c *check.C) { - t := s.Init(c) + test := func(shellCommand string, diagnostics ...string) { + mklines := t.NewMkLines("filename.mk", + "\t"+shellCommand) + ck := NewShellLineChecker(mklines, mklines.mklines[0]) - t.SetUpVartypes() + mklines.ForEach(func(mkline *MkLine) { + ck.CheckShellCommandLine(ck.mkline.ShellCommand()) + }) - test := func(shellWord string, checkQuoting bool, diagnostics ...string) { - // See checkVaruseUndefined and checkVarassignLeftNotUsed. - ck := t.NewShellLineChecker("\t echo " + shellWord) - ck.CheckWord(shellWord, checkQuoting, RunTime) t.CheckOutput(diagnostics) } - // No warning for the outer variable since it is completely indirect. - // The inner variable ${list} must still be defined, though. - test("${${list}}", false, - "WARN: filename.mk:1: list is used but not defined.") + test("@# Comment", + nil...) - // No warning for variables that are partly indirect. - // TODO: Why not? - test("${SED_FILE.${id}}", false, - "WARN: filename.mk:1: id is used but not defined.") + test("uname=`uname`; echo $$uname; echo; ${PREFIX}/bin/command", + "WARN: filename.mk:1: Unknown shell command \"uname\".", + "WARN: filename.mk:1: Please switch to \"set -e\" mode "+ + "before using a semicolon (after \"uname=`uname`\") to separate commands.") - // TODO: Since $@ refers to ${.TARGET} and not sh.argv, there is no point in checking for quotes. - // TODO: Having the same tests for $$@ would be much more interesting. + test("echo ${PKGNAME:Q}", // VucQuotPlain + "NOTE: filename.mk:1: The :Q modifier isn't necessary for ${PKGNAME} here.") - // The unquoted $@ takes a different code path in pkglint than the quoted $@. - test("$@", false, - "WARN: filename.mk:1: Please use \"${.TARGET}\" instead of \"$@\".") + test("echo \"${CFLAGS:Q}\"", // VucQuotDquot - // When $@ appears as part of a shell token, it takes another code path in pkglint. - test("-$@-", false, - "WARN: filename.mk:1: Please use \"${.TARGET}\" instead of \"$@\".") + // ShellLineChecker.checkVaruseToken + "WARN: filename.mk:1: The :Q modifier should not be used inside quotes.", - // The unquoted $@ takes a different code path in pkglint than the quoted $@. - test("\"$@\"", false, - "WARN: filename.mk:1: Please use \"${.TARGET}\" instead of \"$@\".") + // ShellLineChecker.checkVaruseToken + // MkLineChecker.CheckVaruse + // MkLineChecker.checkVarUseQuoting + "WARN: filename.mk:1: Please use ${CFLAGS:M*:Q} instead of ${CFLAGS:Q} "+ + "and make sure the variable appears outside of any quoting characters.") - test("${COMMENT:Q}", true, - nil...) + test("echo '${COMMENT:Q}'", // VucQuotSquot + "WARN: filename.mk:1: The :Q modifier should not be used inside quotes.", + "WARN: filename.mk:1: Please move ${COMMENT:Q} outside of any quoting characters.") - test("\"${DISTINFO_FILE:Q}\"", true, - "NOTE: filename.mk:1: The :Q modifier isn't necessary for ${DISTINFO_FILE} here.") + test("echo target=$@ exitcode=$$? '$$' \"\\$$\"", + "WARN: filename.mk:1: Please use \"${.TARGET}\" instead of \"$@\".", + "WARN: filename.mk:1: The $? shell variable is often not available in \"set -e\" mode.") - test("embed${DISTINFO_FILE:Q}ded", true, - "NOTE: filename.mk:1: The :Q modifier isn't necessary for ${DISTINFO_FILE} here.") + test("echo $$@", + "WARN: filename.mk:1: The $@ shell variable should only be used in double quotes.") - test("s,\\.,,", true, + // No warning about a possibly missed variable name. + // This occurs only rarely, and typically as part of a regular expression + // where it is used intentionally. + test("echo \"$$\"", // As seen by make(1); the shell sees: echo "$" nil...) - test("\"s,\\.,,\"", true, + test("echo \"\\n\"", nil...) -} - -func (s *Suite) Test_ShellLineChecker_CheckWord__dollar_without_variable(c *check.C) { - t := s.Init(c) - - ck := t.NewShellLineChecker("# dummy") - - ck.CheckWord("/.*~$$//g", false, RunTime) // Typical argument to pax(1). - t.CheckOutputEmpty() -} + test("${RUN} for f in *.c; do echo $${f%.c}; done", + nil...) -func (s *Suite) Test_ShellLineChecker_CheckWord__backslash_plus(c *check.C) { - t := s.Init(c) + test("${RUN} set +x; echo $${variable+set}", + nil...) - t.SetUpTool("find", "FIND", AtRunTime) - ck := t.NewShellLineChecker("\tfind . -exec rm -rf {} \\+") + // Based on mail/thunderbird/Makefile, rev. 1.159 + test("${RUN} subdir=\"`unzip -c \"$$e\" install.rdf | awk '/re/ { print \"hello\" }'`\"", + "WARN: filename.mk:1: Double quotes inside backticks inside double quotes are error prone.", + "WARN: filename.mk:1: The exitcode of \"unzip\" at the left of the | operator is ignored.") - ck.CheckShellCommandLine(ck.mkline.ShellCommand()) + // From mail/thunderbird/Makefile, rev. 1.159 + test(""+ + "${RUN} for e in ${XPI_FILES}; do "+ + " subdir=\"`${UNZIP_CMD} -c \"$$e\" install.rdf | "+ + ""+"awk '/.../ {print;exit;}'`\" && "+ + " ${MKDIR} \"${WRKDIR}/extensions/$$subdir\" && "+ + " cd \"${WRKDIR}/extensions/$$subdir\" && "+ + " ${UNZIP_CMD} -aqo $$e; "+ + "done", + "WARN: filename.mk:1: XPI_FILES is used but not defined.", + "WARN: filename.mk:1: Double quotes inside backticks inside double quotes are error prone.", + "WARN: filename.mk:1: The exitcode of \"${UNZIP_CMD}\" at the left of the | operator is ignored.") - // A backslash before any other character than " \ ` is discarded by the parser. - t.CheckOutputEmpty() -} + // From x11/wxGTK28/Makefile + // TODO: Why is TOOLS_PATH.msgfmt not recognized? + // At least, the warning should be more specific, mentioning USE_TOOLS. + test(""+ + "set -e; cd ${WRKSRC}/locale; "+ + "for lang in *.po; do "+ + " [ \"$${lang}\" = \"wxstd.po\" ] && continue; "+ + " ${TOOLS_PATH.msgfmt} -c -o \"$${lang%.po}.mo\" \"$${lang}\"; "+ + "done", + "WARN: filename.mk:1: Unknown shell command \"[\".", + "WARN: filename.mk:1: Unknown shell command \"${TOOLS_PATH.msgfmt}\".") -func (s *Suite) Test_ShellLineChecker_CheckWord__squot_dollar(c *check.C) { - t := s.Init(c) + test("@cp from to", + "WARN: filename.mk:1: The shell command \"cp\" should not be hidden.") - ck := t.NewShellLineChecker("\t'$") + test("-cp from to", + "WARN: filename.mk:1: Using a leading \"-\" to suppress errors is deprecated.") - ck.CheckWord(ck.mkline.ShellCommand(), false, RunTime) + test("-${MKDIR} deeply/nested/subdir", + "WARN: filename.mk:1: Using a leading \"-\" to suppress errors is deprecated.") - // FIXME: Should be parsed correctly. Make passes the dollar through (probably), - // and the shell parser should complain about the unfinished string literal. - t.CheckOutputLines( - "WARN: filename.mk:1: Internal pkglint error in ShTokenizer.ShAtom at \"$\" (quoting=s).", - "WARN: filename.mk:1: Internal pkglint error in ShellLine.CheckWord at \"'$\" (quoting=s), rest: $") -} + G.Pkg = NewPackage(t.File("category/pkgbase")) + G.Pkg.Plist.Dirs["share/pkgbase"] = &PlistLine{ + t.NewLine("PLIST", 123, "share/pkgbase/file"), + nil, + "share/pkgbase/file"} -func (s *Suite) Test_ShellLineChecker_CheckWord__dquot_dollar(c *check.C) { - t := s.Init(c) + // A directory that is found in the PLIST. + // TODO: Add a test for using this command inside a conditional; + // the note should not appear then. + test("${RUN} ${INSTALL_DATA_DIR} share/pkgbase ${PREFIX}/share/pkgbase", + "NOTE: filename.mk:1: You can use AUTO_MKDIRS=yes or \"INSTALLATION_DIRS+= share/pkgbase\" "+ + "instead of \"${INSTALL_DATA_DIR}\".", + "WARN: filename.mk:1: The INSTALL_*_DIR commands can only handle one directory at a time.") - ck := t.NewShellLineChecker("\t\"$") + // A directory that is not found in the PLIST. + test("${RUN} ${INSTALL_DATA_DIR} ${PREFIX}/share/other", + "NOTE: filename.mk:1: You can use \"INSTALLATION_DIRS+= share/other\" instead of \"${INSTALL_DATA_DIR}\".") - ck.CheckWord(ck.mkline.ShellCommand(), false, RunTime) + G.Pkg = nil - // FIXME: Make consumes the dollar silently. - // This could be worth another pkglint warning. - t.CheckOutputEmpty() + // See PR 46570, item "1. It does not" + // No warning about missing error checking ("set -e"). + test("for x in 1 2 3; do echo \"$$x\" || exit 1; done", + nil...) } -func (s *Suite) Test_ShellLineChecker_CheckWord__dollar_subshell(c *check.C) { +// TODO: Document in detail that strip is not a regular tool. +func (s *Suite) Test_ShellLineChecker_CheckShellCommandLine__strip(c *check.C) { t := s.Init(c) - ck := t.NewShellLineChecker("\t$$(echo output)") + test := func(shellCommand string) { + mklines := t.NewMkLines("filename.mk", + "\t"+shellCommand) - ck.CheckWord(ck.mkline.ShellCommand(), false, RunTime) + mklines.ForEach(func(mkline *MkLine) { + ck := NewShellLineChecker(mklines, mkline) + ck.CheckShellCommandLine(mkline.ShellCommand()) + }) + } - t.CheckOutputLines( - "WARN: filename.mk:1: Invoking subshells via $(...) is not portable enough.") -} + test("${STRIP} executable") -func (s *Suite) Test_ShellLineChecker_CheckWord__PKGMANDIR(c *check.C) { - t := s.Init(c) + t.CheckOutputLines( + "WARN: filename.mk:1: Unknown shell command \"${STRIP}\".", + "WARN: filename.mk:1: STRIP is used but not defined.") t.SetUpVartypes() - mklines := t.NewMkLines("chat/ircII/Makefile", - MkCvsID, - "CONFIGURE_ARGS+=--mandir=${DESTDIR}${PREFIX}/man", - "CONFIGURE_ARGS+=--mandir=${DESTDIR}${PREFIX}/${PKGMANDIR}") - mklines.Check() + test("${STRIP} executable") - t.CheckOutputLines( - "WARN: chat/ircII/Makefile:2: Please use ${PKGMANDIR} instead of \"man\".", - "NOTE: chat/ircII/Makefile:2: This variable value should be aligned to column 25.", - "NOTE: chat/ircII/Makefile:3: This variable value should be aligned to column 25.") + t.CheckOutputEmpty() } -func (s *Suite) Test_ShellLineChecker_CheckWord__empty(c *check.C) { +func (s *Suite) Test_ShellLineChecker_CheckShellCommandLine__nofix(c *check.C) { t := s.Init(c) t.SetUpVartypes() - + t.SetUpTool("echo", "", AtRunTime) mklines := t.NewMkLines("Makefile", - MkCvsID, - "", - "JAVA_CLASSPATH=\t# empty") + "\techo ${PKGNAME:Q}") + ck := NewShellLineChecker(mklines, mklines.mklines[0]) - mklines.Check() + ck.CheckShellCommandLine("echo ${PKGNAME:Q}") - t.CheckOutputEmpty() + t.CheckOutputLines( + "NOTE: Makefile:1: The :Q modifier isn't necessary for ${PKGNAME} here.") } -func (s *Suite) Test_ShellLineChecker_unescapeBackticks__unfinished(c *check.C) { +func (s *Suite) Test_ShellLineChecker_CheckShellCommandLine__show_autofix(c *check.C) { t := s.Init(c) - mklines := t.NewMkLines("filename.mk", - MkCvsID, - "", - "pre-configure:", - "\t`${VAR}", // Error in first shell word - "\techo `${VAR}") // Error after first shell word + t.SetUpCommandLine("-Wall", "--show-autofix") + t.SetUpVartypes() + t.SetUpTool("echo", "", AtRunTime) + mklines := t.NewMkLines("Makefile", + "\techo ${PKGNAME:Q}") + ck := NewShellLineChecker(mklines, mklines.mklines[0]) - // Breakpoint in ShellLine.CheckShellCommand - // Breakpoint in ShellLine.CheckToken - // Breakpoint in ShellLine.unescapeBackticks - mklines.Check() + ck.CheckShellCommandLine("echo ${PKGNAME:Q}") t.CheckOutputLines( - "WARN: filename.mk:4: Pkglint ShellLine.CheckShellCommand: splitIntoShellTokens couldn't parse \"`${VAR}\"", - "WARN: filename.mk:5: Pkglint ShellLine.CheckShellCommand: splitIntoShellTokens couldn't parse \"`${VAR}\"") + "NOTE: Makefile:1: The :Q modifier isn't necessary for ${PKGNAME} here.", + "AUTOFIX: Makefile:1: Replacing \"${PKGNAME:Q}\" with \"${PKGNAME}\".") } -func (s *Suite) Test_ShellLineChecker_unescapeBackticks__unfinished_direct(c *check.C) { +func (s *Suite) Test_ShellLineChecker_CheckShellCommandLine__autofix(c *check.C) { t := s.Init(c) - mklines := t.NewMkLines("dummy.mk", - MkCvsID, - "\t# shell command") + t.SetUpCommandLine("-Wall", "--autofix") + t.SetUpVartypes() + t.SetUpTool("echo", "", AtRunTime) + mklines := t.NewMkLines("Makefile", + "\techo ${PKGNAME:Q}") + ck := NewShellLineChecker(mklines, mklines.mklines[0]) - // This call is unrealistic. It doesn't happen in practice, and this - // direct, forcing test is only to reach the code coverage. - atoms := []*ShAtom{ - NewShAtom(shtText, "`", shqBackt)} - NewShellLineChecker(mklines, mklines.mklines[1]). - unescapeBackticks(&atoms, shqBackt) + ck.CheckShellCommandLine("echo ${PKGNAME:Q}") t.CheckOutputLines( - "ERROR: dummy.mk:2: Unfinished backticks after \"\".") + "AUTOFIX: Makefile:1: Replacing \"${PKGNAME:Q}\" with \"${PKGNAME}\".") + + // TODO: There should be a general way of testing a code in the three modes: + // default, --show-autofix, --autofix. } -func (s *Suite) Test_ShellLineChecker_variableNeedsQuoting(c *check.C) { +// TODO: Document the exact purpose of this test, or split it into useful tests. +func (s *Suite) Test_ShellLineChecker_CheckShellCommandLine__implementation(c *check.C) { t := s.Init(c) - test := func(shVarname string, expected bool) { - t.CheckEquals((*ShellLineChecker).variableNeedsQuoting(nil, shVarname), expected) - } + t.SetUpVartypes() + mklines := t.NewMkLines("filename.mk", + "# dummy") + ck := NewShellLineChecker(mklines, mklines.mklines[0]) - test("#", false) // A length is always an integer. - test("?", false) // The exit code is always an integer. - test("$", false) // The PID is always an integer. + // foobar="`echo \"foo bar\"`" + text := "foobar=\"`echo \\\"foo bar\\\"`\"" - // In most cases, file and directory names don't contain special characters, - // and if they do, the package will probably not build. Therefore pkglint - // doesn't require them to be quoted, but doing so does not hurt. - test("d", false) // Typically used for directories. - test("f", false) // Typically used for files. - test("i", false) // Typically used for literal values without special characters. - test("id", false) // Identifiers usually don't use special characters. - test("dir", false) // See d above. - test("file", false) // See f above. - test("src", false) // Typically used when copying files or directories. - test("dst", false) // Typically used when copying files or directories. + tokens, rest := splitIntoShellTokens(dummyLine, text) - test("bindir", false) // A typical GNU-style directory. - test("mandir", false) // A typical GNU-style directory. - test("prefix", false) // + t.CheckDeepEquals(tokens, []string{text}) + t.CheckEquals(rest, "") - test("bindirs", true) // A list of directories is typically separated by spaces. - test("var", true) // Other variables are unknown, so they should be quoted. - test("0", true) // The program name may contain special characters when given as full path. - test("1", true) // Command line arguments can be arbitrary strings. + mklines.ForEach(func(mkline *MkLine) { ck.CheckWord(text, false, RunTime) }) + + t.CheckOutputLines( + "WARN: filename.mk:1: Unknown shell command \"echo\".") + + mklines.ForEach(func(mkline *MkLine) { ck.CheckShellCommandLine(text) }) + + // No parse errors + t.CheckOutputLines( + "WARN: filename.mk:1: Unknown shell command \"echo\".") } -func (s *Suite) Test_ShellLineChecker_variableNeedsQuoting__integration(c *check.C) { +func (s *Suite) Test_ShellLineChecker_CheckShellCommandLine__dollar_without_variable(c *check.C) { t := s.Init(c) t.SetUpVartypes() - t.SetUpTool("cp", "", AtRunTime) + t.SetUpTool("pax", "", AtRunTime) mklines := t.NewMkLines("filename.mk", - MkCvsID, - "", - // It's a bit silly to use shell variables in CONFIGURE_ARGS, - // but as of January 2019 that's the only way to run ShellLine.variableNeedsQuoting. - "CONFIGURE_ARGS+=\t; cp $$dir $$\\# $$target", - "pre-configure:", - "\tcp $$dir $$\\# $$target") + "# dummy") + ck := NewShellLineChecker(mklines, mklines.mklines[0]) - mklines.Check() + ck.CheckShellCommandLine("pax -rwpp -s /.*~$$//g . ${DESTDIR}${PREFIX}") - // As of January 2019, the quoting check is disabled for real shell commands. - // See ShellLine.CheckShellCommand, spc.checkWord. t.CheckOutputLines( - "WARN: filename.mk:3: Unquoted shell variable \"target\".") + "WARN: filename.mk:1: Substitution commands like \"/.*~$$//g\" should always be quoted.") } func (s *Suite) Test_ShellLineChecker_CheckShellCommandLine__echo(c *check.C) { @@ -737,26 +925,6 @@ func (s *Suite) Test_ShellLineChecker_Ch "WARN: Makefile:3: Please use the RCD_SCRIPTS mechanism to install rc.d scripts automatically to ${RCD_SCRIPTS_EXAMPLEDIR}.") } -func (s *Suite) Test_ShellLineChecker_checkInstallCommand(c *check.C) { - t := s.Init(c) - - mklines := t.NewMkLines("filename.mk", - "\t# dummy") - mklines.target = "do-install" - - ck := NewShellLineChecker(mklines, mklines.mklines[0]) - - ck.checkInstallCommand("sed") - - t.CheckOutputLines( - "WARN: filename.mk:1: The shell command \"sed\" should not be used in the install phase.") - - ck.checkInstallCommand("cp") - - t.CheckOutputLines( - "WARN: filename.mk:1: ${CP} should not be used to install files.") -} - func (s *Suite) Test_ShellLineChecker_CheckShellCommandLine__sed_and_mv(c *check.C) { t := s.Init(c) @@ -822,215 +990,47 @@ func (s *Suite) Test_ShellLineChecker_Ch "NOTE: filename.mk:1: You can use \"INSTALLATION_DIRS+= dir2\" instead of \"${INSTALL} -d\".") } -func (s *Suite) Test_ShellLineChecker__shell_comment_with_line_continuation(c *check.C) { +func (s *Suite) Test_ShellLineChecker_checkHiddenAndSuppress(c *check.C) { t := s.Init(c) - mklines := t.SetUpFileMkLines("Makefile", + t.SetUpTool("echo", "ECHO", AtRunTime) + t.SetUpTool("ls", "LS", AtRunTime) + mklines := t.NewMkLines("Makefile", MkCvsID, - "pre-install:", - "\t"+"# comment\\", - "\t"+"echo \"hello\"") - - mklines.Check() - - // TODO: "WARN: ~/Makefile:3--4: A shell comment does not stop at the end of line." - t.CheckOutputEmpty() -} - -func (s *Suite) Test_ShellLineChecker_checkWordQuoting(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - t.SetUpTool("grep", "GREP", AtRunTime) - - test := func(input string, diagnostics ...string) { - mklines := t.NewMkLines("module.mk", - "\t"+input) - ck := NewShellLineChecker(mklines, mklines.mklines[0]) - - ck.checkWordQuoting(ck.mkline.ShellCommand(), true, RunTime) - - t.CheckOutput(diagnostics) - } - - test( - "socklen=`${GREP} 'expr' ${WRKSRC}/config.h`", - nil...) - - test( - "s,$$from,$$to,", - "WARN: module.mk:1: Unquoted shell variable \"from\".", - "WARN: module.mk:1: Unquoted shell variable \"to\".") - - // This variable is typically defined by GNU Configure, - // which cannot handle directories with special characters. - // Therefore using it unquoted is considered safe. - test( - "${PREFIX}/$$bindir/program", - nil...) - - test( - "$$@", - "WARN: module.mk:1: The $@ shell variable should only be used in double quotes.") - - // TODO: Add separate tests for "set +e" and "set -e". - test( - "$$?", - "WARN: module.mk:1: The $? shell variable is often not available in \"set -e\" mode.") - - test( - "$$(cat /bin/true)", - "WARN: module.mk:1: Invoking subshells via $(...) is not portable enough.") - - test( - "\"$$\"", - nil...) - - test( - "$$$$", - nil...) - - test( - "``", - nil...) -} - -func (s *Suite) Test_ShellLineChecker_checkShVarUsePlain__default_warning_level(c *check.C) { - t := s.Init(c) - - t.SetUpCommandLine( /* none */ ) - t.SetUpVartypes() - t.SetUpTool("echo", "", AtRunTime) - - mklines := t.NewMkLines("filename.mk", - MkCvsID, - "CONFIGURE_ARGS+=\techo $$@ $$var", + "", + "show-all-targets: .PHONY", + "\t@echo 'hello'", + "\t@ls -l", + "", + "anything-message: .PHONY", + "\t@echo 'may be hidden'", + "\t@ls 'may be hidden'", "", "pre-configure:", - "\techo $$@ $$var") + "\t@") mklines.Check() - // Using $@ outside of double quotes is so obviously wrong that - // the warning is issued by default. - t.CheckOutputLines( - "WARN: filename.mk:2: The $@ shell variable should only be used in double quotes.", - "WARN: filename.mk:5: The $@ shell variable should only be used in double quotes.") + // No warning about the hidden ls since the target names start + // with "show-" or end with "-message". + t.CheckOutputEmpty() } -func (s *Suite) Test_ShellLineChecker_checkShVarUsePlain__Wall(c *check.C) { +func (s *Suite) Test_ShellLineChecker_checkHiddenAndSuppress__no_tracing(c *check.C) { t := s.Init(c) - t.SetUpVartypes() - t.SetUpTool("echo", "", AtRunTime) - - mklines := t.NewMkLines("filename.mk", + t.SetUpTool("ls", "LS", AtRunTime) + mklines := t.NewMkLines("Makefile", MkCvsID, - "CONFIGURE_ARGS+=\techo $$@ $$var", "", "pre-configure:", - "\techo $$@ $$var") - - mklines.Check() - - // FIXME: It is inconsistent that the check for unquoted shell - // variables is enabled for CONFIGURE_ARGS (where shell variables - // don't make sense at all) but not for real shell commands. - t.CheckOutputLines( - "WARN: filename.mk:2: The $@ shell variable should only be used in double quotes.", - "WARN: filename.mk:2: Unquoted shell variable \"var\".", - "WARN: filename.mk:5: The $@ shell variable should only be used in double quotes.") -} - -func (s *Suite) Test_ShellLineChecker_unescapeBackticks(c *check.C) { - t := s.Init(c) - - test := func(input string, expectedOutput string, expectedRest string, diagnostics ...string) { - ck := t.NewShellLineChecker("# dummy") - - tok := NewShTokenizer(nil, input, false) - atoms := tok.ShAtoms() - - // Set up the correct quoting mode for the test by skipping - // uninteresting atoms at the beginning. - q := shqPlain - for atoms[0].MkText != "`" { - q = atoms[0].Quoting - atoms = atoms[1:] - } - t.CheckEquals(tok.Rest(), "") - - backtCommand := ck.unescapeBackticks(&atoms, q) - - var actualRest strings.Builder - for _, atom := range atoms { - actualRest.WriteString(atom.MkText) - } - - t.CheckEquals(backtCommand, expectedOutput) - t.CheckEquals(actualRest.String(), expectedRest) - t.CheckOutput(diagnostics) - } - - test("`echo`end", "echo", "end") - test("`echo $$var`end", "echo $$var", "end") - test("``end", "", "end") - test("`echo \"hello\"`end", "echo \"hello\"", "end") - test("`echo 'hello'`end", "echo 'hello'", "end") - test("`echo '\\\\\\\\'`end", "echo '\\\\'", "end") - - // Only the characters " $ ` \ are unescaped. All others stay the same. - test("`echo '\\n'`end", "echo '\\n'", "end", - // TODO: Add more details regarding which backslash is meant. - "WARN: filename.mk:1: Backslashes should be doubled inside backticks.") - test("\tsocklen=`${GREP} 'expr' ${WRKSRC}/config.h`", "${GREP} 'expr' ${WRKSRC}/config.h", "") - - // The 2xx test cases are in shqDquot mode. - - test("\"`echo`\"", "echo", "\"") - test("\"`echo \"\"`\"", "echo \"\"", "\"", - "WARN: filename.mk:1: Double quotes inside backticks inside double quotes are error prone.") - - // varname="`echo \"one two\" "\ " "three"`" - test( - "varname=\"`echo \\\"one two\\\" \"\\ \" \"three\"`\"", - "echo \"one two\" \"\\ \" \"three\"", - "\"", - - // TODO: Add more details regarding which backslash and backtick is meant. - "WARN: filename.mk:1: Backslashes should be doubled inside backticks.", - "WARN: filename.mk:1: Double quotes inside backticks inside double quotes are error prone.", - "WARN: filename.mk:1: Double quotes inside backticks inside double quotes are error prone.") -} - -func (s *Suite) Test_ShellLineChecker_unescapeBackticks__dquotBacktDquot(c *check.C) { - t := s.Init(c) - - t.SetUpTool("echo", "", AtRunTime) - mklines := t.NewMkLines("dummy.mk", - MkCvsID, - "\t var=\"`echo \"\"`\"") - - mklines.Check() - - t.CheckOutputLines( - "WARN: dummy.mk:2: Double quotes inside backticks inside double quotes are error prone.") -} - -func (s *Suite) Test_ShellLineChecker__variable_outside_quotes(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - mklines := t.NewMkLines("dummy.mk", - MkCvsID, - "GZIP=\t${ECHO} $$comment") + "\t@ls -l") + t.DisableTracing() mklines.Check() t.CheckOutputLines( - "WARN: dummy.mk:2: The variable GZIP should not be set by any package.", - "WARN: dummy.mk:2: Unquoted shell variable \"comment\".", - "WARN: dummy.mk:2: ECHO should not be used indirectly at load time (via GZIP).") + "WARN: Makefile:4: The shell command \"ls\" should not be hidden.") } func (s *Suite) Test_ShellLineChecker_CheckShellCommand__cd_inside_if(c *check.C) { @@ -1113,565 +1113,584 @@ func (s *Suite) Test_ShellLineChecker_Ch t.CheckOutputEmpty() } -func (s *Suite) Test_ShellLineChecker_checkHiddenAndSuppress(c *check.C) { +func (s *Suite) Test_ShellLineChecker_CheckWord(c *check.C) { t := s.Init(c) - t.SetUpTool("echo", "ECHO", AtRunTime) - t.SetUpTool("ls", "LS", AtRunTime) - mklines := t.NewMkLines("Makefile", - MkCvsID, - "", - "show-all-targets: .PHONY", - "\t@echo 'hello'", - "\t@ls -l", - "", - "anything-message: .PHONY", - "\t@echo 'may be hidden'", - "\t@ls 'may be hidden'", - "", - "pre-configure:", - "\t@") + t.SetUpVartypes() - mklines.Check() + test := func(shellWord string, checkQuoting bool, diagnostics ...string) { + // See checkVaruseUndefined and checkVarassignLeftNotUsed. + ck := t.NewShellLineChecker("\t echo " + shellWord) + ck.CheckWord(shellWord, checkQuoting, RunTime) + t.CheckOutput(diagnostics) + } - // No warning about the hidden ls since the target names start - // with "show-" or end with "-message". - t.CheckOutputEmpty() -} + // No warning for the outer variable since it is completely indirect. + // The inner variable ${list} must still be defined, though. + test("${${list}}", false, + "WARN: filename.mk:1: list is used but not defined.") -func (s *Suite) Test_ShellLineChecker_checkHiddenAndSuppress__no_tracing(c *check.C) { - t := s.Init(c) + // No warning for variables that are partly indirect. + // TODO: Why not? + test("${SED_FILE.${id}}", false, + "WARN: filename.mk:1: id is used but not defined.") - t.SetUpTool("ls", "LS", AtRunTime) - mklines := t.NewMkLines("Makefile", - MkCvsID, - "", - "pre-configure:", - "\t@ls -l") - t.DisableTracing() + // TODO: Since $@ refers to ${.TARGET} and not sh.argv, there is no point in checking for quotes. + // TODO: Having the same tests for $$@ would be much more interesting. - mklines.Check() + // The unquoted $@ takes a different code path in pkglint than the quoted $@. + test("$@", false, + "WARN: filename.mk:1: Please use \"${.TARGET}\" instead of \"$@\".") - t.CheckOutputLines( - "WARN: Makefile:4: The shell command \"ls\" should not be hidden.") -} + // When $@ appears as part of a shell token, it takes another code path in pkglint. + test("-$@-", false, + "WARN: filename.mk:1: Please use \"${.TARGET}\" instead of \"$@\".") -func (s *Suite) Test_SimpleCommandChecker_handleForbiddenCommand(c *check.C) { - t := s.Init(c) + // The unquoted $@ takes a different code path in pkglint than the quoted $@. + test("\"$@\"", false, + "WARN: filename.mk:1: Please use \"${.TARGET}\" instead of \"$@\".") - mklines := t.NewMkLines("Makefile", - MkCvsID, - "", - "\t${RUN} mktexlsr; texconfig") + test("${COMMENT:Q}", true, + nil...) - mklines.Check() + test("\"${DISTINFO_FILE:Q}\"", true, + "NOTE: filename.mk:1: The :Q modifier isn't necessary for ${DISTINFO_FILE} here.") - t.CheckOutputLines( - "ERROR: Makefile:3: \"mktexlsr\" must not be used in Makefiles.", - "ERROR: Makefile:3: \"texconfig\" must not be used in Makefiles.") + test("embed${DISTINFO_FILE:Q}ded", true, + "NOTE: filename.mk:1: The :Q modifier isn't necessary for ${DISTINFO_FILE} here.") + + test("s,\\.,,", true, + nil...) + + test("\"s,\\.,,\"", true, + nil...) } -func (s *Suite) Test_SimpleCommandChecker_handleCommandVariable(c *check.C) { +func (s *Suite) Test_ShellLineChecker_CheckWord__dollar_without_variable(c *check.C) { t := s.Init(c) - t.SetUpTool("perl", "PERL5", AtRunTime) - t.SetUpTool("perl6", "PERL6", Nowhere) - mklines := t.NewMkLines("Makefile", - MkCvsID, - "", - "PERL5_VARS_CMD=\t${PERL5:Q}", - "PERL5_VARS_CMD=\t${PERL6:Q}", - "", - "pre-configure:", - "\t${PERL5_VARS_CMD} -e 'print 12345'") + ck := t.NewShellLineChecker("# dummy") - mklines.Check() + ck.CheckWord("/.*~$$//g", false, RunTime) // Typical argument to pax(1). - // FIXME: In PERL5:Q and PERL6:Q, the :Q is wrong. - t.CheckOutputLines( - "WARN: Makefile:4: The \"${PERL6:Q}\" tool is used but not added to USE_TOOLS.") + t.CheckOutputEmpty() } -func (s *Suite) Test_SimpleCommandChecker_handleCommandVariable__parameterized(c *check.C) { +func (s *Suite) Test_ShellLineChecker_CheckWord__backslash_plus(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/package") - G.Pkg = NewPackage(t.File("category/package")) - t.FinishSetUp() + t.SetUpTool("find", "FIND", AtRunTime) + ck := t.NewShellLineChecker("\tfind . -exec rm -rf {} \\+") - mklines := t.NewMkLines("Makefile", - MkCvsID, - "", - "MY_TOOL.i386=\t${PREFIX}/bin/tool-i386", - "MY_TOOL.x86_64=\t${PREFIX}/bin/tool-x86_64", - "", - "pre-configure:", - "\t${MY_TOOL.amd64} -e 'print 12345'", - "\t${UNKNOWN_TOOL}") + ck.CheckShellCommandLine(ck.mkline.ShellCommand()) - mklines.Check() + // A backslash before any other character than " \ ` is discarded by the parser. + t.CheckOutputEmpty() +} +func (s *Suite) Test_ShellLineChecker_CheckWord__squot_dollar(c *check.C) { + t := s.Init(c) + + ck := t.NewShellLineChecker("\t'$") + + ck.CheckWord(ck.mkline.ShellCommand(), false, RunTime) + + // FIXME: Should be parsed correctly. Make passes the dollar through (probably), + // and the shell parser should complain about the unfinished string literal. t.CheckOutputLines( - "WARN: Makefile:8: Unknown shell command \"${UNKNOWN_TOOL}\".", - "WARN: Makefile:8: UNKNOWN_TOOL is used but not defined.") + "WARN: filename.mk:1: Internal pkglint error in ShTokenizer.ShAtom at \"$\" (quoting=s).", + "WARN: filename.mk:1: Internal pkglint error in ShellLine.CheckWord at \"'$\" (quoting=s), rest: $") } -func (s *Suite) Test_SimpleCommandChecker_handleCommandVariable__followed_by_literal(c *check.C) { +func (s *Suite) Test_ShellLineChecker_CheckWord__dquot_dollar(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/package") - G.Pkg = NewPackage(t.File("category/package")) - t.FinishSetUp() - - mklines := t.NewMkLines("Makefile", - MkCvsID, - "", - "QTDIR=\t${PREFIX}", - "", - "pre-configure:", - "\t${QTDIR}/bin/release") + ck := t.NewShellLineChecker("\t\"$") - mklines.Check() + ck.CheckWord(ck.mkline.ShellCommand(), false, RunTime) + // FIXME: Make consumes the dollar silently. + // This could be worth another pkglint warning. t.CheckOutputEmpty() } -// The package Makefile and other .mk files in a package directory -// may use any shell commands defined by any included files. -// But only if the package is checked as a whole. -// -// On the contrary, when pkglint checks a single .mk file, these -// commands are not guaranteed to be defined, not even when the -// .mk file includes the file defining the command. -// FIXME: This paragraph sounds wrong. All commands from included files should be valid. -// -// The PYTHON_BIN variable below must not be called *_CMD, or another code path is taken. -func (s *Suite) Test_SimpleCommandChecker_handleCommandVariable__from_package(c *check.C) { +func (s *Suite) Test_ShellLineChecker_CheckWord__dollar_subshell(c *check.C) { t := s.Init(c) - pkg := t.SetUpPackage("category/package", - "post-install:", - "\t${PYTHON_BIN}", - "", - ".include \"extra.mk\"") - t.CreateFileLines("category/package/extra.mk", - MkCvsID, - "PYTHON_BIN=\tmy_cmd") - t.FinishSetUp() + ck := t.NewShellLineChecker("\t$$(echo output)") - G.Check(pkg) + ck.CheckWord(ck.mkline.ShellCommand(), false, RunTime) - t.CheckOutputEmpty() + t.CheckOutputLines( + "WARN: filename.mk:1: Invoking subshells via $(...) is not portable enough.") } -// This test ensures that the command line options to INSTALL_*_DIR are properly -// parsed and do not lead to "can only handle one directory at a time" warnings. -func (s *Suite) Test_SimpleCommandChecker_checkInstallMulti(c *check.C) { +func (s *Suite) Test_ShellLineChecker_CheckWord__PKGMANDIR(c *check.C) { t := s.Init(c) t.SetUpVartypes() - mklines := t.NewMkLines("install.mk", + mklines := t.NewMkLines("chat/ircII/Makefile", MkCvsID, - "", - "do-install:", - "\t${INSTALL_PROGRAM_DIR} -m 0555 -g ${APACHE_GROUP} -o ${APACHE_USER} \\", - "\t\t${DESTDIR}${PREFIX}/lib/apache-modules") + "CONFIGURE_ARGS+=--mandir=${DESTDIR}${PREFIX}/man", + "CONFIGURE_ARGS+=--mandir=${DESTDIR}${PREFIX}/${PKGMANDIR}") mklines.Check() t.CheckOutputLines( - "NOTE: install.mk:4--5: You can use \"INSTALLATION_DIRS+= lib/apache-modules\" " + - "instead of \"${INSTALL_PROGRAM_DIR}\".") + "WARN: chat/ircII/Makefile:2: Please use ${PKGMANDIR} instead of \"man\".", + "NOTE: chat/ircII/Makefile:2: This variable value should be aligned to column 25.", + "NOTE: chat/ircII/Makefile:3: This variable value should be aligned to column 25.") } -func (s *Suite) Test_SimpleCommandChecker_checkPaxPe(c *check.C) { +func (s *Suite) Test_ShellLineChecker_CheckWord__empty(c *check.C) { t := s.Init(c) t.SetUpVartypes() - t.SetUpTool("pax", "PAX", AtRunTime) + mklines := t.NewMkLines("Makefile", MkCvsID, "", - "do-install:", - "\t${RUN} pax -pe ${WRKSRC} ${DESTDIR}${PREFIX}", - "\t${RUN} ${PAX} -pe ${WRKSRC} ${DESTDIR}${PREFIX}") + "JAVA_CLASSPATH=\t# empty") mklines.Check() - t.CheckOutputLines( - "WARN: Makefile:4: Please use the -pp option to pax(1) instead of -pe.", - "WARN: Makefile:5: Please use the -pp option to pax(1) instead of -pe.") + t.CheckOutputEmpty() } -func (s *Suite) Test_SimpleCommandChecker_checkEchoN(c *check.C) { +func (s *Suite) Test_ShellLineChecker_checkWordQuoting(c *check.C) { t := s.Init(c) - t.SetUpTool("echo", "ECHO", AtRunTime) - t.SetUpTool("echo -n", "ECHO_N", AtRunTime) - mklines := t.NewMkLines("Makefile", + t.SetUpVartypes() + t.SetUpTool("grep", "GREP", AtRunTime) + + test := func(input string, diagnostics ...string) { + mklines := t.NewMkLines("module.mk", + "\t"+input) + ck := NewShellLineChecker(mklines, mklines.mklines[0]) + + ck.checkWordQuoting(ck.mkline.ShellCommand(), true, RunTime) + + t.CheckOutput(diagnostics) + } + + test( + "socklen=`${GREP} 'expr' ${WRKSRC}/config.h`", + nil...) + + test( + "s,$$from,$$to,", + "WARN: module.mk:1: Unquoted shell variable \"from\".", + "WARN: module.mk:1: Unquoted shell variable \"to\".") + + // This variable is typically defined by GNU Configure, + // which cannot handle directories with special characters. + // Therefore using it unquoted is considered safe. + test( + "${PREFIX}/$$bindir/program", + nil...) + + test( + "$$@", + "WARN: module.mk:1: The $@ shell variable should only be used in double quotes.") + + // TODO: Add separate tests for "set +e" and "set -e". + test( + "$$?", + "WARN: module.mk:1: The $? shell variable is often not available in \"set -e\" mode.") + + test( + "$$(cat /bin/true)", + "WARN: module.mk:1: Invoking subshells via $(...) is not portable enough.") + + test( + "\"$$\"", + nil...) + + test( + "$$$$", + nil...) + + test( + "``", + nil...) +} + +func (s *Suite) Test_ShellLineChecker_unescapeBackticks__unfinished(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("filename.mk", MkCvsID, "", - "do-install:", - "\t${RUN} ${ECHO} -n 'Computing...'", - "\t${RUN} ${ECHO_N} 'Computing...'", - "\t${RUN} ${ECHO} 'Computing...'") + "pre-configure:", + "\t`${VAR}", // Error in first shell word + "\techo `${VAR}") // Error after first shell word + // Breakpoint in ShellLine.CheckShellCommand + // Breakpoint in ShellLine.CheckToken + // Breakpoint in ShellLine.unescapeBackticks mklines.Check() t.CheckOutputLines( - "WARN: Makefile:4: Please use ${ECHO_N} instead of \"echo -n\".") + "WARN: filename.mk:4: Pkglint ShellLine.CheckShellCommand: splitIntoShellTokens couldn't parse \"`${VAR}\"", + "WARN: filename.mk:5: Pkglint ShellLine.CheckShellCommand: splitIntoShellTokens couldn't parse \"`${VAR}\"") } -func (s *Suite) Test_ShellProgramChecker_checkConditionalCd(c *check.C) { +func (s *Suite) Test_ShellLineChecker_unescapeBackticks__unfinished_direct(c *check.C) { t := s.Init(c) - t.SetUpTool("ls", "", AtRunTime) - t.SetUpTool("printf", "", AtRunTime) - t.SetUpTool("wc", "", AtRunTime) - mklines := t.NewMkLines("Makefile", + mklines := t.NewMkLines("dummy.mk", MkCvsID, - "pre-configure:", - "\t${RUN} while cd ..; do printf .; done", - "\t${RUN} while cd .. && cd ..; do printf .; done", // Unusual, therefore no warning. - "\t${RUN} if cd ..; then printf .; fi", - "\t${RUN} ! cd ..", - "\t${RUN} if cd ..; printf 'ok\\n'; then printf .; fi", - "\t${RUN} if cd .. | wc -l; then printf .; fi", // Unusual, therefore no warning. - "\t${RUN} if cd .. && cd ..; then printf .; fi") // Unusual, therefore no warning. + "\t# shell command") - mklines.Check() + // This call is unrealistic. It doesn't happen in practice, and this + // direct, forcing test is only to reach the code coverage. + atoms := []*ShAtom{ + NewShAtom(shtText, "`", shqBackt)} + NewShellLineChecker(mklines, mklines.mklines[1]). + unescapeBackticks(&atoms, shqBackt) t.CheckOutputLines( - "ERROR: Makefile:3: The Solaris /bin/sh cannot handle \"cd\" inside conditionals.", - "ERROR: Makefile:5: The Solaris /bin/sh cannot handle \"cd\" inside conditionals.", - "WARN: Makefile:6: The Solaris /bin/sh does not support negation of shell commands.", - "WARN: Makefile:8: The exitcode of \"cd\" at the left of the | operator is ignored.") + "ERROR: dummy.mk:2: Unfinished backticks after \"\".") } -func (s *Suite) Test_SimpleCommandChecker_checkRegexReplace(c *check.C) { +func (s *Suite) Test_ShellLineChecker_unescapeBackticks(c *check.C) { t := s.Init(c) - test := func(cmd string, diagnostics ...string) { - t.SetUpTool("pax", "PAX", AtRunTime) - t.SetUpTool("sed", "SED", AtRunTime) - mklines := t.NewMkLines("Makefile", - MkCvsID, - "pre-configure:", - "\t"+cmd) - - mklines.Check() + test := func(input string, expectedOutput string, expectedRest string, diagnostics ...string) { + ck := t.NewShellLineChecker("# dummy") - t.CheckOutput(diagnostics) - } + tok := NewShTokenizer(nil, input, false) + atoms := tok.ShAtoms() - test("${PAX} -s s,.*,, src dst", - "WARN: Makefile:3: Substitution commands like \"s,.*,,\" should always be quoted.") + // Set up the correct quoting mode for the test by skipping + // uninteresting atoms at the beginning. + q := shqPlain + for atoms[0].MkText != "`" { + q = atoms[0].Quoting + atoms = atoms[1:] + } + t.CheckEquals(tok.Rest(), "") - test("pax -s s,.*,, src dst", - "WARN: Makefile:3: Substitution commands like \"s,.*,,\" should always be quoted.") + backtCommand := ck.unescapeBackticks(&atoms, q) - test("${SED} -e s,.*,, src dst", - "WARN: Makefile:3: Substitution commands like \"s,.*,,\" should always be quoted.") + var actualRest strings.Builder + for _, atom := range atoms { + actualRest.WriteString(atom.MkText) + } - test("sed -e s,.*,, src dst", - "WARN: Makefile:3: Substitution commands like \"s,.*,,\" should always be quoted.") + t.CheckEquals(backtCommand, expectedOutput) + t.CheckEquals(actualRest.String(), expectedRest) + t.CheckOutput(diagnostics) + } - // The * is properly enclosed in quotes. - test("sed -e 's,.*,,' -e \"s,-*,,\"", - nil...) + test("`echo`end", "echo", "end") + test("`echo $$var`end", "echo $$var", "end") + test("``end", "", "end") + test("`echo \"hello\"`end", "echo \"hello\"", "end") + test("`echo 'hello'`end", "echo 'hello'", "end") + test("`echo '\\\\\\\\'`end", "echo '\\\\'", "end") - // The * is properly escaped. - test("sed -e s,.\\*,,", - nil...) + // Only the characters " $ ` \ are unescaped. All others stay the same. + test("`echo '\\n'`end", "echo '\\n'", "end", + // TODO: Add more details regarding which backslash is meant. + "WARN: filename.mk:1: Backslashes should be doubled inside backticks.") + test("\tsocklen=`${GREP} 'expr' ${WRKSRC}/config.h`", "${GREP} 'expr' ${WRKSRC}/config.h", "") - test("pax -s s,\\.orig,, src dst", - nil...) + // The 2xx test cases are in shqDquot mode. - test("sed -e s,a,b,g src dst", - nil...) + test("\"`echo`\"", "echo", "\"") + test("\"`echo \"\"`\"", "echo \"\"", "\"", + "WARN: filename.mk:1: Double quotes inside backticks inside double quotes are error prone.") - // TODO: Merge the code with BtSedCommands. + // varname="`echo \"one two\" "\ " "three"`" + test( + "varname=\"`echo \\\"one two\\\" \"\\ \" \"three\"`\"", + "echo \"one two\" \"\\ \" \"three\"", + "\"", - // TODO: Finally, remove the G.Testing from the main code. - // Then, remove this test case. - G.Testing = false - test("sed -e s,.*,match,", - nil...) - G.Testing = true + // TODO: Add more details regarding which backslash and backtick is meant. + "WARN: filename.mk:1: Backslashes should be doubled inside backticks.", + "WARN: filename.mk:1: Double quotes inside backticks inside double quotes are error prone.", + "WARN: filename.mk:1: Double quotes inside backticks inside double quotes are error prone.") } -func (s *Suite) Test_SimpleCommandChecker_checkAutoMkdirs(c *check.C) { +func (s *Suite) Test_ShellLineChecker_unescapeBackticks__dquotBacktDquot(c *check.C) { t := s.Init(c) - t.SetUpVartypes() - // TODO: Check whether these tools are actually necessary for this test. - t.SetUpTool("awk", "AWK", AtRunTime) - t.SetUpTool("cp", "CP", AtRunTime) t.SetUpTool("echo", "", AtRunTime) - t.SetUpTool("mkdir", "MKDIR", AtRunTime) // This is actually "mkdir -p". - t.SetUpTool("unzip", "UNZIP_CMD", AtRunTime) - - test := func(shellCommand string, diagnostics ...string) { - mklines := t.NewMkLines("filename.mk", - "\t"+shellCommand) - ck := NewShellLineChecker(mklines, mklines.mklines[0]) - - mklines.ForEach(func(mkline *MkLine) { - ck.CheckShellCommandLine(ck.mkline.ShellCommand()) - }) + mklines := t.NewMkLines("dummy.mk", + MkCvsID, + "\t var=\"`echo \"\"`\"") - t.CheckOutput(diagnostics) - } + mklines.Check() - // AUTO_MKDIRS applies only when installing directories. - test("${RUN} ${INSTALL} -c ${WRKSRC}/file ${PREFIX}/bin/", - nil...) + t.CheckOutputLines( + "WARN: dummy.mk:2: Double quotes inside backticks inside double quotes are error prone.") +} - // TODO: Warn that ${INSTALL} -d can only handle a single directory. - test("${RUN} ${INSTALL} -m 0755 -d ${PREFIX}/first ${PREFIX}/second", - "NOTE: filename.mk:1: You can use \"INSTALLATION_DIRS+= first\" instead of \"${INSTALL} -d\".", - "NOTE: filename.mk:1: You can use \"INSTALLATION_DIRS+= second\" instead of \"${INSTALL} -d\".") +func (s *Suite) Test_ShellLineChecker_checkShVarUsePlain__default_warning_level(c *check.C) { + t := s.Init(c) - G.Pkg = NewPackage(t.File("category/pkgbase")) - G.Pkg.Plist.Dirs["share/pkgbase"] = &PlistLine{ - t.NewLine("PLIST", 123, "share/pkgbase/file"), - nil, - "share/pkgbase/file"} + t.SetUpCommandLine( /* none */ ) + t.SetUpVartypes() + t.SetUpTool("echo", "", AtRunTime) - // A directory that is found in the PLIST. - // TODO: Add a test for using this command inside a conditional; - // the note should not appear then. - test("${RUN} ${INSTALL_DATA_DIR} share/pkgbase ${PREFIX}/share/pkgbase", - "NOTE: filename.mk:1: You can use AUTO_MKDIRS=yes or \"INSTALLATION_DIRS+= share/pkgbase\" "+ - "instead of \"${INSTALL_DATA_DIR}\".", - "WARN: filename.mk:1: The INSTALL_*_DIR commands can only handle one directory at a time.") + mklines := t.NewMkLines("filename.mk", + MkCvsID, + "CONFIGURE_ARGS+=\techo $$@ $$var", + "", + "pre-configure:", + "\techo $$@ $$var") - // Directories from .for loops are too dynamic to be replaced with AUTO_MKDIRS. - // TODO: Expand simple .for loops. - test("${RUN} ${INSTALL_DATA_DIR} ${PREFIX}/${dir}", - "WARN: filename.mk:1: dir is used but not defined.") + mklines.Check() - // A directory that is not found in the PLIST would not be created by AUTO_MKDIRS, - // therefore only INSTALLATION_DIRS is suggested. - test("${RUN} ${INSTALL_DATA_DIR} ${PREFIX}/share/other", - "NOTE: filename.mk:1: You can use \"INSTALLATION_DIRS+= share/other\" instead of \"${INSTALL_DATA_DIR}\".") + // Using $@ outside of double quotes is so obviously wrong that + // the warning is issued by default. + t.CheckOutputLines( + "WARN: filename.mk:2: The $@ shell variable should only be used in double quotes.", + "WARN: filename.mk:5: The $@ shell variable should only be used in double quotes.") } -func (s *Suite) Test_SimpleCommandChecker_checkAutoMkdirs__redundant(c *check.C) { +func (s *Suite) Test_ShellLineChecker_checkShVarUsePlain__Wall(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/package", - "AUTO_MKDIRS=\t\tyes", - "INSTALLATION_DIRS+=\tshare/redundant", - "INSTALLATION_DIRS+=\tnot/redundant ${EGDIR}") - t.CreateFileLines("category/package/PLIST", - PlistCvsID, - "share/redundant/file", - "${EGDIR}/file") + t.SetUpVartypes() + t.SetUpTool("echo", "", AtRunTime) - t.Main("-Wall", "-q", "category/package") + mklines := t.NewMkLines("filename.mk", + MkCvsID, + "CONFIGURE_ARGS+=\techo $$@ $$var", + "", + "pre-configure:", + "\techo $$@ $$var") + + mklines.Check() + // FIXME: It is inconsistent that the check for unquoted shell + // variables is enabled for CONFIGURE_ARGS (where shell variables + // don't make sense at all) but not for real shell commands. t.CheckOutputLines( - "NOTE: ~/category/package/Makefile:21: The directory \"share/redundant\" "+ - "is redundant in INSTALLATION_DIRS.", - // The below is not proven to be always correct. It assumes that a - // variable in the Makefile has the same value as the corresponding - // variable from PLIST_SUBST. Violating this assumption would be - // confusing to the pkgsrc developers, therefore it's a safe bet. - // A notable counterexample is PKGNAME in PLIST, which corresponds - // to PKGNAME_NOREV in the package Makefile. - "NOTE: ~/category/package/Makefile:22: The directory \"${EGDIR}\" "+ - "is redundant in INSTALLATION_DIRS.") + "WARN: filename.mk:2: The $@ shell variable should only be used in double quotes.", + "WARN: filename.mk:2: Unquoted shell variable \"var\".", + "WARN: filename.mk:5: The $@ shell variable should only be used in double quotes.") } -// The AUTO_MKDIRS code in mk/install/install.mk (install-dirs-from-PLIST) -// skips conditional directories, as well as directories with placeholders. -func (s *Suite) Test_SimpleCommandChecker_checkAutoMkdirs__conditional_PLIST(c *check.C) { +func (s *Suite) Test_ShellLineChecker_variableNeedsQuoting(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/package", - "LIB_SUBDIR=\tsubdir", - "", - "do-install:", - "\t${RUN} ${INSTALL_DATA_DIR} ${PREFIX}/libexec/always", - "\t${RUN} ${INSTALL_DATA_DIR} ${PREFIX}/libexec/conditional", - "\t${RUN} ${INSTALL_DATA_DIR} ${PREFIX}/${LIB_SUBDIR}", - ) - t.Chdir("category/package") - t.CreateFileLines("PLIST", - PlistCvsID, - "libexec/always/always", - "${LIB_SUBDIR}/file", - "${PLIST.cond}libexec/conditional/conditional") - t.FinishSetUp() + test := func(shVarname string, expected bool) { + t.CheckEquals((*ShellLineChecker).variableNeedsQuoting(nil, shVarname), expected) + } + + test("#", false) // A length is always an integer. + test("?", false) // The exit code is always an integer. + test("$", false) // The PID is always an integer. + + // In most cases, file and directory names don't contain special characters, + // and if they do, the package will probably not build. Therefore pkglint + // doesn't require them to be quoted, but doing so does not hurt. + test("d", false) // Typically used for directories. + test("f", false) // Typically used for files. + test("i", false) // Typically used for literal values without special characters. + test("id", false) // Identifiers usually don't use special characters. + test("dir", false) // See d above. + test("file", false) // See f above. + test("src", false) // Typically used when copying files or directories. + test("dst", false) // Typically used when copying files or directories. - G.checkdirPackage(".") + test("bindir", false) // A typical GNU-style directory. + test("mandir", false) // A typical GNU-style directory. + test("prefix", false) // - // As libexec/conditional will not be created automatically, - // AUTO_MKDIRS must not be suggested in that line. - t.CheckOutputLines( - "NOTE: Makefile:23: You can use AUTO_MKDIRS=yes "+ - "or \"INSTALLATION_DIRS+= libexec/always\" "+ - "instead of \"${INSTALL_DATA_DIR}\".", - "NOTE: Makefile:24: You can use "+ - "\"INSTALLATION_DIRS+= libexec/conditional\" "+ - "instead of \"${INSTALL_DATA_DIR}\".", - "NOTE: Makefile:25: You can use "+ - "\"INSTALLATION_DIRS+= ${LIB_SUBDIR}\" "+ - "instead of \"${INSTALL_DATA_DIR}\".") + test("bindirs", true) // A list of directories is typically separated by spaces. + test("var", true) // Other variables are unknown, so they should be quoted. + test("0", true) // The program name may contain special characters when given as full path. + test("1", true) // Command line arguments can be arbitrary strings. + test("comment", true) // Comments can be arbitrary strings. } -func (s *Suite) Test_ShellProgramChecker_checkSetE__simple_commands(c *check.C) { +func (s *Suite) Test_ShellLineChecker_variableNeedsQuoting__integration(c *check.C) { t := s.Init(c) - t.SetUpTool("echo", "", AtRunTime) - t.SetUpTool("rm", "", AtRunTime) - t.SetUpTool("touch", "", AtRunTime) - mklines := t.NewMkLines("Makefile", + t.SetUpVartypes() + t.SetUpTool("cp", "", AtRunTime) + mklines := t.NewMkLines("filename.mk", MkCvsID, + "", + // It's a bit silly to use shell variables in CONFIGURE_ARGS, + // but as of January 2019 that's the only way to run ShellLine.variableNeedsQuoting. + "CONFIGURE_ARGS+=\t; cp $$dir $$\\# $$target", "pre-configure:", - "\techo 1; echo 2; echo 3", - "\techo 1; touch file; rm file", - "\techo 1; var=value; echo 3") + "\tcp $$dir $$\\# $$target") mklines.Check() + // As of January 2019, the quoting check is disabled for real shell commands. + // See ShellLine.CheckShellCommand, spc.checkWord. t.CheckOutputLines( - "WARN: Makefile:4: Please switch to \"set -e\" mode before using a semicolon " + - "(after \"touch file\") to separate commands.") + "WARN: filename.mk:3: Unquoted shell variable \"target\".") } -func (s *Suite) Test_ShellProgramChecker_checkSetE__compound_commands(c *check.C) { +func (s *Suite) Test_ShellLineChecker_checkInstallCommand(c *check.C) { t := s.Init(c) - t.SetUpTool("echo", "", AtRunTime) - t.SetUpTool("touch", "", AtRunTime) - mklines := t.NewMkLines("Makefile", - MkCvsID, - "pre-configure:", - "\ttouch file; for f in file; do echo \"$$f\"; done", - "\tfor f in file; do echo \"$$f\"; done; touch file", - "\ttouch 1; touch 2; touch 3; touch 4") + mklines := t.NewMkLines("filename.mk", + "\t# dummy") + mklines.target = "do-install" - mklines.Check() + ck := NewShellLineChecker(mklines, mklines.mklines[0]) + + ck.checkInstallCommand("sed") t.CheckOutputLines( - "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon "+ - "(after \"touch file\") to separate commands.", - "WARN: Makefile:5: Please switch to \"set -e\" mode before using a semicolon "+ - "(after \"touch 1\") to separate commands.") + "WARN: filename.mk:1: The shell command \"sed\" should not be used in the install phase.") + + ck.checkInstallCommand("cp") + + t.CheckOutputLines( + "WARN: filename.mk:1: ${CP} should not be used to install files.") } -func (s *Suite) Test_ShellProgramChecker_checkSetE__no_tracing(c *check.C) { +func (s *Suite) Test_splitIntoShellTokens__line_continuation(c *check.C) { t := s.Init(c) - t.SetUpTool("touch", "", AtRunTime) - mklines := t.NewMkLines("Makefile", - MkCvsID, - "pre-configure:", - "\ttouch 1; touch 2") - t.DisableTracing() + words, rest := splitIntoShellTokens(dummyLine, "if true; then \\") - mklines.Check() + t.CheckDeepEquals(words, []string{"if", "true", ";", "then"}) + t.CheckEquals(rest, "\\") t.CheckOutputLines( - "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon " + - "(after \"touch 1\") to separate commands.") + "WARN: Internal pkglint error in ShTokenizer.ShAtom at \"\\\\\" (quoting=plain).") } -func (s *Suite) Test_ShellProgramChecker_canFail(c *check.C) { +func (s *Suite) Test_splitIntoShellTokens__dollar_slash(c *check.C) { t := s.Init(c) - t.SetUpVartypes() - t.SetUpTool("basename", "", AtRunTime) - t.SetUpTool("dirname", "", AtRunTime) - t.SetUpTool("echo", "", AtRunTime) - t.SetUpTool("env", "", AtRunTime) - t.SetUpTool("grep", "GREP", AtRunTime) - t.SetUpTool("sed", "", AtRunTime) - t.SetUpTool("touch", "", AtRunTime) - t.SetUpTool("tr", "tr", AtRunTime) - t.SetUpTool("true", "TRUE", AtRunTime) + words, rest := splitIntoShellTokens(dummyLine, "pax -s /.*~$$//g") - test := func(cmd string, diagnostics ...string) { - mklines := t.NewMkLines("Makefile", - MkCvsID, - "pre-configure:", - "\t"+cmd+" ; echo 'done.'") + t.CheckDeepEquals(words, []string{"pax", "-s", "/.*~$$//g"}) + t.CheckEquals(rest, "") +} - mklines.Check() +func (s *Suite) Test_splitIntoShellTokens__dollar_subshell(c *check.C) { + t := s.Init(c) - t.CheckOutput(diagnostics) - } + words, rest := splitIntoShellTokens(dummyLine, "id=$$(${AWK} '{print}' < ${WRKSRC}/idfile) && echo \"$$id\"") - test("socklen=`${GREP} 'expr' ${WRKSRC}/config.h`", - "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon "+ - "(after \"socklen=`${GREP} 'expr' ${WRKSRC}/config.h`\") to separate commands.") + t.CheckDeepEquals(words, []string{"id=$$(${AWK} '{print}' < ${WRKSRC}/idfile)", "&&", "echo", "\"$$id\""}) + t.CheckEquals(rest, "") +} - test("socklen=`${GREP} 'expr' ${WRKSRC}/config.h || ${TRUE}`", - nil...) +func (s *Suite) Test_splitIntoShellTokens__semicolons(c *check.C) { + t := s.Init(c) - test("socklen=$$(expr 16)", - "WARN: Makefile:3: Invoking subshells via $(...) is not portable enough.", - "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon "+ - "(after \"socklen=$$(expr 16)\") to separate commands.") + words, rest := splitIntoShellTokens(dummyLine, "word1 word2;;;") - test("socklen=$$(expr 16 || true)", - "WARN: Makefile:3: Invoking subshells via $(...) is not portable enough.") + t.CheckDeepEquals(words, []string{"word1", "word2", ";;", ";"}) + t.CheckEquals(rest, "") +} - test("socklen=$$(expr 16 || ${TRUE})", - "WARN: Makefile:3: Invoking subshells via $(...) is not portable enough.") +func (s *Suite) Test_splitIntoShellTokens__whitespace(c *check.C) { + t := s.Init(c) - test("${ECHO_MSG} \"Message\"", - nil...) + text := "\t${RUN} cd ${WRKSRC}&&(${ECHO} ${PERL5:Q};${ECHO})|${BASH} ./install" + words, rest := splitIntoShellTokens(dummyLine, text) - test("${FAIL_MSG} \"Failure\"", - "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon "+ - "(after \"${FAIL_MSG} \\\"Failure\\\"\") to separate commands.") + t.CheckDeepEquals(words, []string{ + "${RUN}", + "cd", "${WRKSRC}", + "&&", "(", "${ECHO}", "${PERL5:Q}", ";", "${ECHO}", ")", + "|", "${BASH}", "./install"}) + t.CheckEquals(rest, "") +} - test("set -x", - "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon "+ - "(after \"set -x\") to separate commands.") +func (s *Suite) Test_splitIntoShellTokens__finished_dquot(c *check.C) { + t := s.Init(c) - test("echo 'input' | sed -e s,in,out,", - nil...) + text := "\"\"" + words, rest := splitIntoShellTokens(dummyLine, text) - test("sed -e s,in,out,", - nil...) + t.CheckDeepEquals(words, []string{"\"\""}) + t.CheckEquals(rest, "") +} - test("sed s,in,out,", - nil...) +func (s *Suite) Test_splitIntoShellTokens__unfinished_dquot(c *check.C) { + t := s.Init(c) - test("grep input", - nil...) + text := "\t\"" + words, rest := splitIntoShellTokens(dummyLine, text) - test("grep pattern file...", - "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon "+ - "(after \"grep pattern file...\") to separate commands.") + c.Check(words, check.IsNil) + t.CheckEquals(rest, "\"") +} - test("touch file", - "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon "+ - "(after \"touch file\") to separate commands.") +func (s *Suite) Test_splitIntoShellTokens__unescaped_dollar_in_dquot(c *check.C) { + t := s.Init(c) - test("echo 'starting'", - nil...) + text := "echo \"$$\"" + words, rest := splitIntoShellTokens(dummyLine, text) - test("echo 'logging' > log", - "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon "+ - "(after \"echo 'logging'\") to separate commands.") + t.CheckDeepEquals(words, []string{"echo", "\"$$\""}) + t.CheckEquals(rest, "") - test("echo 'to stderr' 1>&2", - nil...) + t.CheckOutputEmpty() +} - test("echo 'hello' | tr -d 'aeiou'", - nil...) +func (s *Suite) Test_splitIntoShellTokens__varuse_with_embedded_space_and_other_vars(c *check.C) { + t := s.Init(c) - test("env | grep '^PATH='", - nil...) + varuseWord := "${GCONF_SCHEMAS:@.s.@${INSTALL_DATA} ${WRKSRC}/src/common/dbus/${.s.} ${DESTDIR}${GCONF_SCHEMAS_DIR}/@}" + words, rest := splitIntoShellTokens(dummyLine, varuseWord) - test("basename dir/file", - nil...) + t.CheckDeepEquals(words, []string{varuseWord}) + t.CheckEquals(rest, "") +} - test("dirname dir/file", - nil...) +// Two shell variables, next to each other, +// are two separate atoms but count as a single token. +func (s *Suite) Test_splitIntoShellTokens__two_shell_variables(c *check.C) { + t := s.Init(c) + + code := "echo $$i$$j" + words, rest := splitIntoShellTokens(dummyLine, code) + + t.CheckDeepEquals(words, []string{"echo", "$$i$$j"}) + t.CheckEquals(rest, "") +} + +func (s *Suite) Test_splitIntoShellTokens__varuse_with_embedded_space(c *check.C) { + t := s.Init(c) + + words, rest := splitIntoShellTokens(dummyLine, "${VAR:S/ /_/g}") + + t.CheckDeepEquals(words, []string{"${VAR:S/ /_/g}"}) + t.CheckEquals(rest, "") +} + +func (s *Suite) Test_splitIntoShellTokens__redirect(c *check.C) { + t := s.Init(c) + + words, rest := splitIntoShellTokens(dummyLine, "echo 1>output 2>>append 3>|clobber 4>&5 6>append") + + t.CheckDeepEquals(words, []string{ + "echo", + "1>", "output", + "2>>", "append", + "3>|", "clobber", + "4>&", "5", + "6<", "input", + ">>", "append"}) + t.CheckEquals(rest, "") + + words, rest = splitIntoShellTokens(dummyLine, "echo 1> output 2>> append 3>| clobber 4>& 5 6< input >> append") + + t.CheckDeepEquals(words, []string{ + "echo", + "1>", "output", + "2>>", "append", + "3>|", "clobber", + "4>&", "5", + "6<", "input", + ">>", "append"}) + t.CheckEquals(rest, "") }