Merge pull request #5479 from benlangmuir/compact-doc-struct

[sourcekit] Introduce a lazy representation for DocumentStructure responses

Author: benlangmuir

tools/SourceKit/tools/sourcekitd/include/sourcekitd/CompactArray.h

@@ -21,7 +21,9 @@ namespace sourcekitd {
 class CompactArrayBuilderImpl {
 public:
   std::unique_ptr<llvm::MemoryBuffer> createBuffer() const;
+  void appendTo(llvm::SmallVectorImpl<char> &Buf) const;
 
+  size_t sizeInBytes() const;
   bool empty() const;
 
 protected:
@@ -35,30 +37,12 @@ class CompactArrayBuilderImpl {
 
 private:
   unsigned getOffsetForString(llvm::StringRef Str);
+  void copyInto(char *BufPtr, size_t Length) const;
 
   llvm::SmallVector<uint8_t, 256> EntriesBuffer;
   llvm::SmallString<256> StringBuffer;
 };
 
-template <typename ...EntryTypes>
-class CompactArrayBuilder : public CompactArrayBuilderImpl {
-public:
-  void addEntry(EntryTypes... Args) {
-    add(Args...);
-  }
-
-private:
-  template <typename T, typename ...Targs>
-  void add(T Val, Targs... Args) {
-    add(Val);
-    add(Args...);
-  }
-
-  template <typename T>
-  void add(T Val) {
-    addImpl(Val);
-  }
-};
 
 template <typename T>
 struct CompactArrayField {
@@ -88,6 +72,31 @@ struct CompactArrayEntriesSizer<T> {
   }
 };
 
+template <typename ...EntryTypes>
+class CompactArrayBuilder : public CompactArrayBuilderImpl {
+public:
+  void addEntry(EntryTypes... Args) {
+    add(Args...);
+    count += 1;
+  }
+
+  size_t size() const { return count; }
+
+private:
+  template <typename T, typename ...Targs>
+  void add(T Val, Targs... Args) {
+    add(Val);
+    add(Args...);
+  }
+
+  template <typename T>
+  void add(T Val) {
+    addImpl(Val);
+  }
+
+  size_t count = 0;
+};
+
 class CompactArrayReaderImpl {
 protected:
   CompactArrayReaderImpl(void *Buf) : Buf(Buf) {}

tools/SourceKit/tools/sourcekitd/include/sourcekitd/DocStructureArray.h

@@ -0,0 +1,55 @@
+//===--- DocStructureArray.h - ----------------------------------*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See http://swift.org/LICENSE.txt for license information
+// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SOURCEKITD_DOC_STRUCTURE_ARRAY_H
+#define LLVM_SOURCEKITD_DOC_STRUCTURE_ARRAY_H
+
+#include "sourcekitd/Internal.h"
+#include "llvm/ADT/SmallString.h"
+
+namespace sourcekitd {
+
+VariantFunctions *getVariantFunctionsForDocStructureArray();
+VariantFunctions *getVariantFunctionsForDocStructureElementArray();
+VariantFunctions *getVariantFunctionsForInheritedTypesArray();
+VariantFunctions *getVariantFunctionsForAttributesArray();
+
+class DocStructureArrayBuilder {
+public:
+  DocStructureArrayBuilder();
+  ~DocStructureArrayBuilder();
+
+  void beginSubStructure(unsigned Offset, unsigned Length,
+                         SourceKit::UIdent Kind, SourceKit::UIdent AccessLevel,
+                         SourceKit::UIdent SetterAccessLevel,
+                         unsigned NameOffset, unsigned NameLength,
+                         unsigned BodyOffset, unsigned BodyLength,
+                         llvm::StringRef DisplayName, llvm::StringRef TypeName,
+                         llvm::StringRef RuntimeName,
+                         llvm::StringRef SelectorName,
+                         llvm::ArrayRef<llvm::StringRef> InheritedTypes,
+                         llvm::ArrayRef<SourceKit::UIdent> Attrs);
+
+  void addElement(SourceKit::UIdent Kind, unsigned Offset, unsigned Length);
+
+  void endSubStructure();
+
+  std::unique_ptr<llvm::MemoryBuffer> createBuffer();
+
+private:
+  struct Implementation;
+  Implementation &impl;
+};
+
+} // end namespace sourcekitd
+
+#endif // LLVM_SOURCEKITD_DOC_STRUCTURE_ARRAY_H

tools/SourceKit/tools/sourcekitd/include/sourcekitd/Internal.h

@@ -47,6 +47,10 @@ enum class CustomBufferKind {
   TokenAnnotationsArray,
   DocSupportAnnotationArray,
   CodeCompletionResultsArray,
+  DocStructureArray,
+  InheritedTypesArray,
+  DocStructureElementArray,
+  AttributesArray,
 };
 
 class ResponseBuilder {

tools/SourceKit/tools/sourcekitd/lib/API/CMakeLists.txt

@@ -1,6 +1,7 @@
 set(sourcekitdAPI_sources
   CodeCompletionResultsArray.cpp
   CompactArray.cpp
+  DocStructureArray.cpp
   DocSupportAnnotationArray.cpp
   Requests.cpp
   sourcekitdAPI-Common.cpp

tools/SourceKit/tools/sourcekitd/lib/API/CompactArray.cpp

@@ -43,8 +43,12 @@ void CompactArrayBuilderImpl::addImpl(StringRef Val) {
 }
 
 void CompactArrayBuilderImpl::addImpl(UIdent Val) {
-  sourcekitd_uid_t uid = SKDUIDFromUIdent(Val);
-  addScalar(uid, EntriesBuffer);
+  if (Val.isValid()) {
+    sourcekitd_uid_t uid = SKDUIDFromUIdent(Val);
+    addScalar(uid, EntriesBuffer);
+  } else {
+    addScalar(sourcekitd_uid_t(nullptr), EntriesBuffer);
+  }
 }
 
 void CompactArrayBuilderImpl::addImpl(Optional<llvm::StringRef> Val) {
@@ -67,27 +71,37 @@ unsigned CompactArrayBuilderImpl::getOffsetForString(StringRef Str) {
 
 std::unique_ptr<llvm::MemoryBuffer>
 CompactArrayBuilderImpl::createBuffer() const {
-  size_t EntriesBufSize = EntriesBuffer.size();
   std::unique_ptr<llvm::MemoryBuffer> Buf;
-  Buf = llvm::MemoryBuffer::getNewUninitMemBuffer(
-      sizeof(uint64_t) +
-      EntriesBufSize +
-      StringBuffer.size());
-
-  char *BufPtr = (char*)Buf->getBufferStart();
-  *reinterpret_cast<uint64_t*>(BufPtr) = EntriesBufSize;
-  BufPtr += sizeof(uint64_t);
+  Buf = llvm::MemoryBuffer::getNewUninitMemBuffer(sizeInBytes());
+  copyInto(const_cast<char *>(Buf->getBufferStart()), Buf->getBufferSize());
+  return Buf;
+}
+
+void CompactArrayBuilderImpl::appendTo(SmallVectorImpl<char> &Buf) const {
+  size_t OrigSize = Buf.size();
+  size_t NewSize = OrigSize + sizeInBytes();
+  Buf.resize(NewSize);
+  copyInto(Buf.data() + OrigSize, NewSize - OrigSize);
+}
+
+void CompactArrayBuilderImpl::copyInto(char *BufPtr, size_t Length) const {
+  uint64_t EntriesBufSize = EntriesBuffer.size();
+  assert(Length >= sizeInBytes());
+  memcpy(BufPtr, &EntriesBufSize, sizeof(EntriesBufSize));
+  BufPtr += sizeof(EntriesBufSize);
   memcpy(BufPtr, EntriesBuffer.data(), EntriesBufSize);
   BufPtr += EntriesBufSize;
   memcpy(BufPtr, StringBuffer.data(), StringBuffer.size());
-
-  return Buf;
 }
 
 bool CompactArrayBuilderImpl::empty() const {
   return EntriesBuffer.empty();
 }
 
+size_t CompactArrayBuilderImpl::sizeInBytes() const {
+  return sizeof(uint64_t) + EntriesBuffer.size() + StringBuffer.size();
+}
+
 template <typename T>
 static void readScalar(const uint8_t *Buf, T &Val) {
   memcpy(&Val, Buf, sizeof(Val));