[WIP/Review] Add basic CUDA support (for CMake projects)

src/indexer.cc

@@ -81,7 +81,8 @@ struct IndexParam {
 
       if (!vfs.Stamp(path, it->second.mtime, no_linkage ? 3 : 1))
         return;
-      it->second.db = std::make_unique<IndexFile>(path, it->second.content);
+      it->second.db =
+          std::make_unique<IndexFile>(path, it->second.content, no_linkage);
     }
   }
 
@@ -1173,11 +1174,12 @@ class IndexFrontendAction : public ASTFrontendAction {
 };
 } // namespace
 
-const int IndexFile::kMajorVersion = 20;
+const int IndexFile::kMajorVersion = 21;
 const int IndexFile::kMinorVersion = 0;
 
-IndexFile::IndexFile(const std::string &path, const std::string &contents)
-    : path(path), file_contents(contents) {}
+IndexFile::IndexFile(const std::string &path, const std::string &contents,
+                     bool no_linkage)
+    : path(path), no_linkage(no_linkage), file_contents(contents) {}
 
 IndexFunc &IndexFile::ToFunc(Usr usr) {
   auto [it, inserted] = usr2func.try_emplace(usr);
@@ -1236,8 +1238,8 @@ Index(SemaManager *manager, WorkingFiles *wfiles, VFS *vfs,
   ok = false;
   // -fparse-all-comments enables documentation in the indexer and in
   // code completion.
-  if (g_config->index.comments > 1)
-    CI->getLangOpts()->CommentOpts.ParseAllComments = true;
+  CI->getLangOpts()->CommentOpts.ParseAllComments = g_config->index.comments > 1;
+  CI->getLangOpts()->RetainCommentsFromSystemHeaders = true;
   std::string buf = wfiles->GetContent(main);
   std::vector<std::unique_ptr<llvm::MemoryBuffer>> Bufs;
   if (buf.size())
@@ -1249,12 +1251,18 @@ Index(SemaManager *manager, WorkingFiles *wfiles, VFS *vfs,
   DiagnosticConsumer DC;
   auto Clang = std::make_unique<CompilerInstance>(PCH);
   Clang->setInvocation(std::move(CI));
-  Clang->setVirtualFileSystem(FS);
   Clang->createDiagnostics(&DC, false);
   Clang->setTarget(TargetInfo::CreateTargetInfo(
       Clang->getDiagnostics(), Clang->getInvocation().TargetOpts));
   if (!Clang->hasTarget())
     return {};
+  Clang->getPreprocessorOpts().RetainRemappedFileBuffers = true;
+#if LLVM_VERSION_MAJOR >= 9 // rC357037
+  Clang->createFileManager(FS);
+#else
+  Clang->setVirtualFileSystem(FS);
+  Clang->createFileManager();
+#endif
 
   IndexParam param(*vfs, no_linkage);
   auto DataConsumer = std::make_shared<IndexDataConsumer>(param);
@@ -1266,6 +1274,7 @@ Index(SemaManager *manager, WorkingFiles *wfiles, VFS *vfs,
     IndexOpts.IndexFunctionLocals = true;
     IndexOpts.IndexImplicitInstantiation = true;
 #if LLVM_VERSION_MAJOR >= 9
+
     IndexOpts.IndexParametersInDeclarations =
         g_config->index.parametersInDeclarations;
     IndexOpts.IndexTemplateParameters = true;
@@ -1294,8 +1303,6 @@ Index(SemaManager *manager, WorkingFiles *wfiles, VFS *vfs,
     LOG_S(ERROR) << "failed to index " << main;
     return {};
   }
-  for (auto &Buf : Bufs)
-    Buf.release();
 
   std::vector<std::unique_ptr<IndexFile>> result;
   for (auto &it : param.UID2File) {

src/indexer.hh

@@ -304,6 +304,7 @@ struct IndexFile {
   // This is unfortunately time_t as used by clang::FileEntry
   int64_t mtime = 0;
   LanguageId language = LanguageId::C;
+  bool no_linkage;
 
   // uid2lid_and_path is used to generate lid2path, but not serialized.
   std::unordered_map<llvm::sys::fs::UniqueID, std::pair<int, std::string>>
@@ -328,7 +329,8 @@ struct IndexFile {
   // File contents at the time of index. Not serialized.
   std::string file_contents;
 
-  IndexFile(const std::string &path, const std::string &contents);
+  IndexFile(const std::string &path, const std::string &contents,
+            bool no_linkage);
 
   IndexFunc &ToFunc(Usr usr);
   IndexType &ToType(Usr usr);

src/messages/initialize.cc

@@ -226,8 +226,8 @@ void Reflect(JsonReader &reader, InitializeParam::Trace &value) {
     value = InitializeParam::Trace::Verbose;
 }
 
-REFLECT_STRUCT(InitializeParam, rootUri, initializationOptions, capabilities,
-               trace, workspaceFolders);
+// initializationOptions is deserialized separately.
+REFLECT_STRUCT(InitializeParam, rootUri, capabilities, trace, workspaceFolders);
 
 struct InitializeResult {
   ServerCap capabilities;
@@ -395,6 +395,17 @@ void Initialize(MessageHandler *m, InitializeParam &param, ReplyOnce &reply) {
 void MessageHandler::initialize(JsonReader &reader, ReplyOnce &reply) {
   InitializeParam param;
   Reflect(reader, param);
+  auto it = reader.m->FindMember("initializationOptions");
+  if (it != reader.m->MemberEnd() && it->value.IsObject()) {
+    JsonReader m1(&it->value);
+    try {
+      Reflect(m1, param.initializationOptions);
+    } catch (std::invalid_argument &) {
+      reader.path_.push_back("initializationOptions");
+      reader.path_.insert(reader.path_.end(), m1.path_.begin(), m1.path_.end());
+      throw;
+    }
+  }
   if (!param.rootUri) {
     reply.Error(ErrorCode::InvalidRequest, "expected rootUri");
     return;

src/pipeline.cc

@@ -298,7 +298,10 @@ bool Indexer_Parse(SemaManager *completion, WorkingFiles *wfiles,
                            request.mode != IndexMode::Background);
       {
         std::lock_guard lock1(vfs->mutex);
-        vfs->state[path_to_index].loaded++;
+        VFS::State &st = vfs->state[path_to_index];
+        st.loaded++;
+        if (prev->no_linkage)
+          st.step = 2;
       }
       lock.unlock();
 
@@ -317,6 +320,8 @@ bool Indexer_Parse(SemaManager *completion, WorkingFiles *wfiles,
             continue;
           st.loaded++;
           st.timestamp = prev->mtime;
+          if (prev->no_linkage)
+            st.step = 3;
         }
         IndexUpdate update = IndexUpdate::CreateDelta(nullptr, prev.get());
         on_indexed->PushBack(std::move(update),
@@ -341,9 +346,9 @@ bool Indexer_Parse(SemaManager *completion, WorkingFiles *wfiles,
 
   std::vector<std::unique_ptr<IndexFile>> indexes;
   if (deleted) {
-    indexes.push_back(std::make_unique<IndexFile>(request.path, ""));
+    indexes.push_back(std::make_unique<IndexFile>(request.path, "", false));
     if (request.path != path_to_index)
-      indexes.push_back(std::make_unique<IndexFile>(path_to_index, ""));
+      indexes.push_back(std::make_unique<IndexFile>(path_to_index, "", false));
   } else {
     std::vector<std::pair<std::string, std::string>> remapped;
     if (g_config->index.onChange) {

src/query.cc

@@ -128,7 +128,7 @@ QueryType::Def Convert(const IndexType::Def &o) {
 
 IndexUpdate IndexUpdate::CreateDelta(IndexFile *previous, IndexFile *current) {
   IndexUpdate r;
-  static IndexFile empty(current->path, "<empty>");
+  static IndexFile empty(current->path, "<empty>", false);
   if (previous)
     r.prev_lid2path = std::move(previous->lid2path);
   else

src/sema_manager.cc

@@ -308,16 +308,21 @@ std::unique_ptr<CompilerInstance> BuildCompilerInstance(
 
   auto Clang = std::make_unique<CompilerInstance>(session.PCH);
   Clang->setInvocation(std::move(CI));
-  Clang->setVirtualFileSystem(FS);
   Clang->createDiagnostics(&DC, false);
   Clang->setTarget(TargetInfo::CreateTargetInfo(
       Clang->getDiagnostics(), Clang->getInvocation().TargetOpts));
   if (!Clang->hasTarget())
     return nullptr;
+  Clang->getPreprocessorOpts().RetainRemappedFileBuffers = true;
   // Construct SourceManager with UserFilesAreVolatile: true because otherwise
   // RequiresNullTerminator: true may cause out-of-bounds read when a file is
   // mmap'ed but is saved concurrently.
+#if LLVM_VERSION_MAJOR >= 9 // rC357037
+  Clang->createFileManager(FS);
+#else
+  Clang->setVirtualFileSystem(FS);
   Clang->createFileManager();
+#endif
   Clang->setSourceManager(new SourceManager(Clang->getDiagnostics(),
                                             Clang->getFileManager(), true));
   auto &IS = Clang->getFrontendOpts().Inputs;
@@ -358,6 +363,7 @@ void BuildPreamble(Session &session, CompilerInvocation &CI,
   CI.getDiagnosticOpts().IgnoreWarnings = false;
   CI.getFrontendOpts().SkipFunctionBodies = true;
   CI.getLangOpts()->CommentOpts.ParseAllComments = g_config->index.comments > 1;
+  CI.getLangOpts()->RetainCommentsFromSystemHeaders = true;
 
   StoreDiags DC(task.path);
   IntrusiveRefCntPtr<DiagnosticsEngine> DE =
@@ -489,7 +495,6 @@ void *CompletionMain(void *manager_) {
     Clang->setCodeCompletionConsumer(task->Consumer.release());
     if (!Parse(*Clang))
       continue;
-    Buf.release();
 
     task->on_complete(&Clang->getCodeCompletionConsumer());
   }
@@ -579,7 +584,6 @@ void *DiagnosticMain(void *manager_) {
       continue;
     if (!Parse(*Clang))
       continue;
-    Buf.release();
 
     auto Fill = [](const DiagBase &d, Diagnostic &ret) {
       ret.range = lsRange{{d.range.start.line, d.range.start.column},

src/serializer.cc

@@ -374,6 +374,7 @@ template <typename TVisitor> void Reflect1(TVisitor &vis, IndexFile &v) {
   if (!gTestOutputMode) {
     REFLECT_MEMBER(mtime);
     REFLECT_MEMBER(language);
+    REFLECT_MEMBER(no_linkage);
     REFLECT_MEMBER(lid2path);
     REFLECT_MEMBER(import_file);
     REFLECT_MEMBER(args);
@@ -407,6 +408,11 @@ void Reflect(JsonWriter &vis, SerializeFormat &v) {
   }
 }
 
+void ReflectMemberStart(JsonReader &vis) {
+  if (!vis.m->IsObject())
+    throw std::invalid_argument("object");
+}
+
 static BumpPtrAllocator Alloc;
 static DenseSet<CachedHashStringRef> Strings;
 static std::mutex AllocMutex;
@@ -482,7 +488,7 @@ Deserialize(SerializeFormat format, const std::string &path,
       if (major != IndexFile::kMajorVersion ||
           minor != IndexFile::kMinorVersion)
         throw std::invalid_argument("Invalid version");
-      file = std::make_unique<IndexFile>(path, file_content);
+      file = std::make_unique<IndexFile>(path, file_content, false);
       ReflectFile(reader, *file);
     } catch (std::invalid_argument &e) {
       LOG_S(INFO) << "failed to deserialize '" << path << "': " << e.what();
@@ -505,7 +511,7 @@ Deserialize(SerializeFormat format, const std::string &path,
     if (reader.HasParseError())
       return nullptr;
 
-    file = std::make_unique<IndexFile>(path, file_content);
+    file = std::make_unique<IndexFile>(path, file_content, false);
     JsonReader json_reader{&reader};
     try {
       ReflectFile(json_reader, *file);

src/serializer.hh

@@ -379,6 +379,7 @@ template <typename T> void Reflect(BinaryWriter &vis, std::vector<T> &v) {
 
 // ReflectMember
 
+void ReflectMemberStart(JsonReader &);
 template <typename T> void ReflectMemberStart(T &) {}
 inline void ReflectMemberStart(JsonWriter &vis) { vis.StartObject(); }