llvm.org GIT mirror llvm / dc103f8
[Support] Explicitly detect recursive response files Previous detection relied upon an arbitrary hard coded limit of 21 response files, which some code bases were running up against. The new detection maintains a stack of processing response files and explicitly checks if a newly encountered file is in the current stack. Some bookkeeping data is necessary in order to detect when to pop the stack. Patch by Chris Glover. Differential Revision: https://reviews.llvm.org/D62798 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@363005 91177308-0d34-0410-b5e6-96231b3b80d8 Shoaib Meenai a month ago
2 changed file(s) with 118 addition(s) and 29 deletion(s). Raw diff Collapse all Expand all
10961096 bool cl::ExpandResponseFiles(StringSaver &Saver, TokenizerCallback Tokenizer,
10971097 SmallVectorImpl &Argv,
10981098 bool MarkEOLs, bool RelativeNames) {
1099 unsigned ExpandedRspFiles = 0;
11001099 bool AllExpanded = true;
1100 struct ResponseFileRecord {
1101 const char *File;
1102 size_t End;
1103 };
1104
1105 // To detect recursive response files, we maintain a stack of files and the
1106 // position of the last argument in the file. This position is updated
1107 // dynamically as we recursively expand files.
1108 SmallVector FileStack;
1109
1110 // Push a dummy entry that represents the initial command line, removing
1111 // the need to check for an empty list.
1112 FileStack.push_back({"", Argv.size()});
11011113
11021114 // Don't cache Argv.size() because it can change.
11031115 for (unsigned I = 0; I != Argv.size();) {
1116 while (I == FileStack.back().End) {
1117 // Passing the end of a file's argument list, so we can remove it from the
1118 // stack.
1119 FileStack.pop_back();
1120 }
1121
11041122 const char *Arg = Argv[I];
11051123 // Check if it is an EOL marker
11061124 if (Arg == nullptr) {
11071125 ++I;
11081126 continue;
11091127 }
1128
11101129 if (Arg[0] != '@') {
11111130 ++I;
11121131 continue;
11131132 }
11141133
1115 // If we have too many response files, leave some unexpanded. This avoids
1116 // crashing on self-referential response files.
1117 if (ExpandedRspFiles > 20)
1118 return false;
1134 const char *FName = Arg + 1;
1135 auto IsEquivalent = [FName](const ResponseFileRecord &RFile) {
1136 return sys::fs::equivalent(RFile.File, FName);
1137 };
1138
1139 // Check for recursive response files.
1140 if (std::any_of(FileStack.begin() + 1, FileStack.end(), IsEquivalent)) {
1141 // This file is recursive, so we leave it in the argument stream and
1142 // move on.
1143 AllExpanded = false;
1144 ++I;
1145 continue;
1146 }
11191147
11201148 // Replace this response file argument with the tokenization of its
11211149 // contents. Nested response files are expanded in subsequent iterations.
11221150 SmallVector ExpandedArgv;
1123 if (ExpandResponseFile(Arg + 1, Saver, Tokenizer, ExpandedArgv, MarkEOLs,
1124 RelativeNames)) {
1125 ++ExpandedRspFiles;
1126 } else {
1151 if (!ExpandResponseFile(FName, Saver, Tokenizer, ExpandedArgv, MarkEOLs,
1152 RelativeNames)) {
11271153 // We couldn't read this file, so we leave it in the argument stream and
11281154 // move on.
11291155 AllExpanded = false;
11301156 ++I;
11311157 continue;
11321158 }
1159
1160 for (ResponseFileRecord &Record : FileStack) {
1161 // Increase the end of all active records by the number of newly expanded
1162 // arguments, minus the response file itself.
1163 Record.End += ExpandedArgv.size() - 1;
1164 }
1165
1166 FileStack.push_back({FName, I + ExpandedArgv.size()});
11331167 Argv.erase(Argv.begin() + I);
11341168 Argv.insert(Argv.begin() + I, ExpandedArgv.begin(), ExpandedArgv.end());
11351169 }
1170
1171 // If successful, the top of the file stack will mark the end of the Argv
1172 // stream. A failure here indicates a bug in the stack popping logic above.
1173 // Note that FileStack may have more than one element at this point because we
1174 // don't have a chance to pop the stack when encountering recursive files at
1175 // the end of the stream, so seeing that doesn't indicate a bug.
1176 assert(FileStack.size() > 0 && Argv.size() == FileStack.back().End);
11361177 return AllExpanded;
11371178 }
11381179
789789 EXPECT_TRUE(IncludedFile.is_open());
790790 IncludedFile << "-option_1 -option_2\n"
791791 "@incdir/resp2\n"
792 "-option_3=abcd\n";
792 "-option_3=abcd\n"
793 "@incdir/resp3\n"
794 "-option_4=efjk\n";
793795 IncludedFile.close();
794796
795797 // Directory for included file.
807809 IncludedFile2 << "-option_23=abcd\n";
808810 IncludedFile2.close();
809811
812 // Create second included response file of second level.
813 llvm::SmallString<128> IncludedFileName3;
814 llvm::sys::path::append(IncludedFileName3, IncDir, "resp3");
815 std::ofstream IncludedFile3(IncludedFileName3.c_str());
816 EXPECT_TRUE(IncludedFile3.is_open());
817 IncludedFile3 << "-option_31 -option_32\n";
818 IncludedFile3 << "-option_33=abcd\n";
819 IncludedFile3.close();
820
810821 // Prepare 'file' with reference to response file.
811822 SmallString<128> IncRef;
812823 IncRef.append(1, '@');
820831 bool Res = llvm::cl::ExpandResponseFiles(
821832 Saver, llvm::cl::TokenizeGNUCommandLine, Argv, false, true);
822833 EXPECT_TRUE(Res);
823 EXPECT_EQ(Argv.size(), 9U);
834 EXPECT_EQ(Argv.size(), 13U);
824835 EXPECT_STREQ(Argv[0], "test/test");
825836 EXPECT_STREQ(Argv[1], "-flag_1");
826837 EXPECT_STREQ(Argv[2], "-option_1");
829840 EXPECT_STREQ(Argv[5], "-option_22");
830841 EXPECT_STREQ(Argv[6], "-option_23=abcd");
831842 EXPECT_STREQ(Argv[7], "-option_3=abcd");
832 EXPECT_STREQ(Argv[8], "-flag_2");
833
843 EXPECT_STREQ(Argv[8], "-option_31");
844 EXPECT_STREQ(Argv[9], "-option_32");
845 EXPECT_STREQ(Argv[10], "-option_33=abcd");
846 EXPECT_STREQ(Argv[11], "-option_4=efjk");
847 EXPECT_STREQ(Argv[12], "-flag_2");
848
849 llvm::sys::fs::remove(IncludedFileName3);
834850 llvm::sys::fs::remove(IncludedFileName2);
835851 llvm::sys::fs::remove(IncDir);
836852 llvm::sys::fs::remove(IncludedFileName);
842858 std::error_code EC = sys::fs::createUniqueDirectory("unittest", TestDir);
843859 EXPECT_TRUE(!EC);
844860
845 SmallString<128> ResponseFilePath;
846 sys::path::append(ResponseFilePath, TestDir, "recursive.rsp");
847 std::string ResponseFileRef = std::string("@") + ResponseFilePath.c_str();
848
849 std::ofstream ResponseFile(ResponseFilePath.str());
850 EXPECT_TRUE(ResponseFile.is_open());
851 ResponseFile << ResponseFileRef << "\n";
852 ResponseFile << ResponseFileRef << "\n";
853 ResponseFile.close();
854
855 // Ensure the recursive expansion terminates.
856 SmallVector Argv = {"test/test", ResponseFileRef.c_str()};
861 SmallString<128> SelfFilePath;
862 sys::path::append(SelfFilePath, TestDir, "self.rsp");
863 std::string SelfFileRef = std::string("@") + SelfFilePath.c_str();
864
865 SmallString<128> NestedFilePath;
866 sys::path::append(NestedFilePath, TestDir, "nested.rsp");
867 std::string NestedFileRef = std::string("@") + NestedFilePath.c_str();
868
869 SmallString<128> FlagFilePath;
870 sys::path::append(FlagFilePath, TestDir, "flag.rsp");
871 std::string FlagFileRef = std::string("@") + FlagFilePath.c_str();
872
873 std::ofstream SelfFile(SelfFilePath.str());
874 EXPECT_TRUE(SelfFile.is_open());
875 SelfFile << "-option_1\n";
876 SelfFile << FlagFileRef << "\n";
877 SelfFile << NestedFileRef << "\n";
878 SelfFile << SelfFileRef << "\n";
879 SelfFile.close();
880
881 std::ofstream NestedFile(NestedFilePath.str());
882 EXPECT_TRUE(NestedFile.is_open());
883 NestedFile << "-option_2\n";
884 NestedFile << FlagFileRef << "\n";
885 NestedFile << SelfFileRef << "\n";
886 NestedFile << NestedFileRef << "\n";
887 NestedFile.close();
888
889 std::ofstream FlagFile(FlagFilePath.str());
890 EXPECT_TRUE(FlagFile.is_open());
891 FlagFile << "-option_x\n";
892 FlagFile.close();
893
894 // Ensure:
895 // Recursive expansion terminates
896 // Recursive files never expand
897 // Non-recursive repeats are allowed
898 SmallVector Argv = {"test/test", SelfFileRef.c_str(),
899 "-option_3"};
857900 BumpPtrAllocator A;
858901 StringSaver Saver(A);
859902 #ifdef _WIN32
864907 bool Res = cl::ExpandResponseFiles(Saver, Tokenizer, Argv, false, false);
865908 EXPECT_FALSE(Res);
866909
867 // Ensure some expansion took place.
868 EXPECT_GT(Argv.size(), 2U);
910 EXPECT_EQ(Argv.size(), 9U);
869911 EXPECT_STREQ(Argv[0], "test/test");
870 for (size_t i = 1; i < Argv.size(); ++i)
871 EXPECT_STREQ(Argv[i], ResponseFileRef.c_str());
912 EXPECT_STREQ(Argv[1], "-option_1");
913 EXPECT_STREQ(Argv[2], "-option_x");
914 EXPECT_STREQ(Argv[3], "-option_2");
915 EXPECT_STREQ(Argv[4], "-option_x");
916 EXPECT_STREQ(Argv[5], SelfFileRef.c_str());
917 EXPECT_STREQ(Argv[6], NestedFileRef.c_str());
918 EXPECT_STREQ(Argv[7], SelfFileRef.c_str());
919 EXPECT_STREQ(Argv[8], "-option_3");
872920 }
873921
874922 TEST(CommandLineTest, ResponseFilesAtArguments) {