Define TORCHTEXT_API macro for visibility control (#1806)

## Context:

TorchText uses dual-binding (PyBind11 and TorchBind) to make custom operations available in Python.
The both binding eventually calls the same implementation contained in `libtorchtext.so`.
The ones bound via PyBind11 (the ones in `torchtext._torchtext`) calls into `libtorchtext.so`.

![Untitled drawing](https://user-images.githubusercontent.com/855818/175428489-c288b3cc-0b9f-4230-95ed-fd7c063bb6fa.jpg)

This means that `libtorchtext.so` has to make the symbols (APIs) used by `torchtext._torchtext` visible.

However, the default visibility of symbols in shared libraries are different in Windows.
On Windows all the symbols are by default hidden. To work around this, we use `CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS` to expose all the symbols.
There is an upper limit of visible symbols one library fine can contain, and this can be problematic in the future.
(Although it is unlikely that torchtext will hit the limit, unless it introduces custom CUDA kernels.)

A better approach is to selectively mark the symbols that should be visible as visible.

## Summary of the change set

This commit introduces `TORCHTEXT_API` macro which annotates functions with proper visibility.
The core logic was taken from https://github.com/pytorch/pytorch/blob/bcc02769bef1d7b89bec724223284958b7c5b564/c10/macros/Export.h

The behavior is as follow;

For non-Windows: It is always `__attribute__((__visibility__("default")))`
For Windows:
If the header is included from the compilation unit of `libtorchtext`, then it resolves to `__declspec(dllexport)`. otherwise it resolves to `__declspec(dllimport)`.

This allows to remove `CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS`.

Author: mthrok

torchtext/csrc/CMakeLists.txt

@@ -1,9 +1,3 @@
-# the following line is added in order to export symbols when building on Windows
-# this approach has some limitations as documented in https://github.com/pytorch/pytorch/pull/3650
-if (MSVC)
-  set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
-endif()
-
 ################################################################################
 # libtorchtext
 ################################################################################
@@ -41,7 +35,8 @@ set(
   )
 
 set(
-  LIBTORCHTEXT_COMPILE_DEFINITIONS)
+  LIBTORCHTEXT_COMPILE_DEFINITIONS
+  TORCHTEXT_BUILD_MAIN_LIB)
 
 function (define_library name source include_dirs link_libraries compile_defs)
   add_library(${name} SHARED ${source})

torchtext/csrc/bert_tokenizer.h

@@ -1,3 +1,4 @@
+#include <torchtext/csrc/export.h>
 #include <torchtext/csrc/vocab.h>
 #include <string>
 #include <vector>
@@ -11,19 +12,20 @@ typedef std::tuple<bool, c10::optional<bool>, std::vector<std::string>>
     BERTEncoderStates;
 
 struct BERTEncoder : torch::CustomClassHolder {
-  BERTEncoder(
+  TORCHTEXT_API BERTEncoder(
       const std::string& vocab_file,
       bool do_lower_case,
       c10::optional<bool> strip_accents);
   BERTEncoder(
       Vocab vocab,
       bool do_lower_case,
       c10::optional<bool> strip_accents);
-  std::vector<std::string> Tokenize(std::string text);
-  std::vector<int64_t> Encode(std::string text);
-  std::vector<std::vector<std::string>> BatchTokenize(
+  TORCHTEXT_API std::vector<std::string> Tokenize(std::string text);
+  TORCHTEXT_API std::vector<int64_t> Encode(std::string text);
+  TORCHTEXT_API std::vector<std::vector<std::string>> BatchTokenize(
+      std::vector<std::string> text);
+  TORCHTEXT_API std::vector<std::vector<int64_t>> BatchEncode(
       std::vector<std::string> text);
-  std::vector<std::vector<int64_t>> BatchEncode(std::vector<std::string> text);
 
   Vocab vocab_;
   bool do_lower_case_;
@@ -40,8 +42,8 @@ struct BERTEncoder : torch::CustomClassHolder {
   static std::string kUnkToken;
 };
 
-BERTEncoderStates _serialize_bert_encoder(
-    const c10::intrusive_ptr<BERTEncoder>& self);
-c10::intrusive_ptr<BERTEncoder> _deserialize_bert_encoder(
+TORCHTEXT_API BERTEncoderStates
+_serialize_bert_encoder(const c10::intrusive_ptr<BERTEncoder>& self);
+TORCHTEXT_API c10::intrusive_ptr<BERTEncoder> _deserialize_bert_encoder(
     BERTEncoderStates states);
 } // namespace torchtext

torchtext/csrc/clip_tokenizer.h

@@ -1,6 +1,7 @@
 #ifndef CLIP_TOKENIZER_H_
 #define CLIP_TOKENIZER_H_
 
+#include <torchtext/csrc/export.h>
 #include <torchtext/csrc/gpt2_bpe_tokenizer.h>
 
 namespace torchtext {
@@ -25,21 +26,22 @@ struct CLIPEncoder : GPT2BPEEncoder {
  public:
   using GPT2BPEEncoder::GPT2BPEEncoder;
 
-  std::vector<int64_t> Encode(const std::string& text);
-  std::vector<std::string> Tokenize(const std::string& text);
+  TORCHTEXT_API std::vector<int64_t> Encode(const std::string& text);
+  TORCHTEXT_API std::vector<std::string> Tokenize(const std::string& text);
 
  protected:
-  std::vector<std::string> BPE_(
+  TORCHTEXT_API std::vector<std::string> BPE_(
       const std::vector<std::string>& token_list) override;
 
-  std::vector<std::string> PreTokenize_(std::string input) override;
+  TORCHTEXT_API std::vector<std::string> PreTokenize_(
+      std::string input) override;
 };
 
-CLIPEncoderStatesPybind _serialize_clip_encoder_pybind(
-    const c10::intrusive_ptr<CLIPEncoder>& self);
+TORCHTEXT_API CLIPEncoderStatesPybind
+_serialize_clip_encoder_pybind(const c10::intrusive_ptr<CLIPEncoder>& self);
 CLIPEncoderStatesTorchbind _serialize_clip_encoder_torchbind(
     const c10::intrusive_ptr<CLIPEncoder>& self);
-c10::intrusive_ptr<CLIPEncoder> _deserialize_clip_encoder_pybind(
+TORCHTEXT_API c10::intrusive_ptr<CLIPEncoder> _deserialize_clip_encoder_pybind(
     CLIPEncoderStatesPybind states);
 c10::intrusive_ptr<CLIPEncoder> _deserialize_clip_encoder_torchbind(
     CLIPEncoderStatesTorchbind states);

torchtext/csrc/export.h

@@ -0,0 +1,35 @@
+#pragma once
+
+// Define the visibility of symbols.
+// The original logic and background can be found here.
+// https://github.com/pytorch/pytorch/blob/bcc02769bef1d7b89bec724223284958b7c5b564/c10/macros/Export.h#L49-L55
+//
+// In the context of torchtext, the logic is simpler at the moment.
+//
+// The torchtext custom operations are implemented in
+// `torchtext/lib/libtorchtext.[so|pyd]`. Some symbols are referred from
+// `torchtext._torchtext`.
+//
+// In Windows, default visibility of dynamically library are hidden, while in
+// Linux/macOS, they are visible.
+//
+// At the moment we do not expect torchtext libraries to be built/linked
+// statically. We assume they are always shared.
+
+#ifdef _WIN32
+#define TORCHTEXT_EXPORT __declspec(dllexport)
+#define TORCHTEXT_IMPORT __declspec(dllimport)
+#else // _WIN32
+#if defined(__GNUC__)
+#define TORCHTEXT_EXPORT __attribute__((__visibility__("default")))
+#else // defined(__GNUC__)
+#define TORCHTEXT_EXPORT
+#endif // defined(__GNUC__)
+#define TORCHTEXT_IMPORT TORCHTEXT_EXPORT
+#endif // _WIN32
+
+#ifdef TORCHTEXT_BUILD_MAIN_LIB
+#define TORCHTEXT_API TORCHTEXT_EXPORT
+#else
+#define TORCHTEXT_API TORCHTEXT_IMPORT
+#endif

torchtext/csrc/gpt2_bpe_tokenizer.h

@@ -2,6 +2,7 @@
 #define GPT2_BPE_TOKENIZER_H_
 
 #include <torch/script.h>
+#include <torchtext/csrc/export.h>
 
 #include <cstdint>
 #include <string>
@@ -79,7 +80,7 @@ struct GPT2BPEEncoder : torch::CustomClassHolder {
       const c10::Dict<int64_t, std::string>& byte_encoder,
       bool caching_enabled = false);
 
-  explicit GPT2BPEEncoder(
+  TORCHTEXT_API explicit GPT2BPEEncoder(
       const std::unordered_map<std::string, int64_t>& bpe_encoder,
       const std::unordered_map<std::string, int64_t>& bpe_merge_ranks,
       const std::string& seperator,
@@ -97,20 +98,21 @@ struct GPT2BPEEncoder : torch::CustomClassHolder {
   //  --> bpe encode --> bpe token ids: [707, 5927], [11], [707, 68]
   //  --> result --> [707, 5927, 11, 707, 68]
   //
-  std::vector<int64_t> Encode(const std::string& text);
-  std::vector<std::string> Tokenize(const std::string& text);
+  TORCHTEXT_API std::vector<int64_t> Encode(const std::string& text);
+  TORCHTEXT_API std::vector<std::string> Tokenize(const std::string& text);
 
-  std::unordered_map<std::string, int64_t> GetBPEEncoder() const;
-  std::unordered_map<std::string, int64_t> GetBPEMergeRanks() const;
-  std::unordered_map<int64_t, std::string> GetByteEncoder() const;
+  TORCHTEXT_API std::unordered_map<std::string, int64_t> GetBPEEncoder() const;
+  TORCHTEXT_API std::unordered_map<std::string, int64_t> GetBPEMergeRanks()
+      const;
+  TORCHTEXT_API std::unordered_map<int64_t, std::string> GetByteEncoder() const;
 };
 
-GPT2BPEEncoderStatesPybind _serialize_gpt2_bpe_encoder_pybind(
+TORCHTEXT_API GPT2BPEEncoderStatesPybind _serialize_gpt2_bpe_encoder_pybind(
     const c10::intrusive_ptr<GPT2BPEEncoder>& self);
 GPT2BPEEncoderStatesTorchbind _serialize_gpt2_bpe_encoder_torchbind(
     const c10::intrusive_ptr<GPT2BPEEncoder>& self);
-c10::intrusive_ptr<GPT2BPEEncoder> _deserialize_gpt2_bpe_encoder_pybind(
-    GPT2BPEEncoderStatesPybind states);
+TORCHTEXT_API c10::intrusive_ptr<GPT2BPEEncoder>
+_deserialize_gpt2_bpe_encoder_pybind(GPT2BPEEncoderStatesPybind states);
 c10::intrusive_ptr<GPT2BPEEncoder> _deserialize_gpt2_bpe_encoder_torchbind(
     GPT2BPEEncoderStatesTorchbind states);
 } // namespace torchtext

torchtext/csrc/regex.h

@@ -1,6 +1,7 @@
 #include <re2/re2.h>
 #include <re2/stringpiece.h>
 #include <torch/script.h>
+#include <torchtext/csrc/export.h>
 #include <string>
 
 namespace torchtext {
@@ -11,12 +12,14 @@ struct Regex : torch::CustomClassHolder {
  public:
   std::string re_str_;
 
-  Regex(const std::string& re_str);
-  std::string Sub(std::string str, const std::string& repl) const;
-  bool FindAndConsume(re2::StringPiece* input, std::string* text) const;
+  TORCHTEXT_API Regex(const std::string& re_str);
+  TORCHTEXT_API std::string Sub(std::string str, const std::string& repl) const;
+  TORCHTEXT_API bool FindAndConsume(re2::StringPiece* input, std::string* text)
+      const;
 };
 
-std::string _serialize_regex(const c10::intrusive_ptr<Regex>& self);
-c10::intrusive_ptr<Regex> _deserialize_regex(std::string&& state);
+TORCHTEXT_API std::string _serialize_regex(
+    const c10::intrusive_ptr<Regex>& self);
+TORCHTEXT_API c10::intrusive_ptr<Regex> _deserialize_regex(std::string&& state);
 
 } // namespace torchtext

torchtext/csrc/regex_tokenizer.h

@@ -1,5 +1,6 @@
 #include <re2/re2.h>
 #include <torch/script.h>
+#include <torchtext/csrc/export.h>
 
 namespace torchtext {
 
@@ -19,16 +20,16 @@ struct RegexTokenizer : torch::CustomClassHolder {
   std::vector<std::string> replacements_;
   bool to_lower_;
 
-  explicit RegexTokenizer(
+  TORCHTEXT_API explicit RegexTokenizer(
       const std::vector<std::string>& patterns,
       const std::vector<std::string>& replacements,
       const bool to_lower);
-  std::vector<std::string> forward(std::string str) const;
+  TORCHTEXT_API std::vector<std::string> forward(std::string str) const;
 };
 
-RegexTokenizerStates _serialize_regex_tokenizer(
-    const c10::intrusive_ptr<RegexTokenizer>& self);
-c10::intrusive_ptr<RegexTokenizer> _deserialize_regex_tokenizer(
+TORCHTEXT_API RegexTokenizerStates
+_serialize_regex_tokenizer(const c10::intrusive_ptr<RegexTokenizer>& self);
+TORCHTEXT_API c10::intrusive_ptr<RegexTokenizer> _deserialize_regex_tokenizer(
     RegexTokenizerStates&& states);
 
 } // namespace torchtext

torchtext/csrc/sentencepiece.h

@@ -1,6 +1,7 @@
 #include <sentencepiece_processor.h>
 #include <sentencepiece_trainer.h>
 #include <torch/script.h>
+#include <torchtext/csrc/export.h>
 
 namespace torchtext {
 
@@ -16,16 +17,19 @@ struct SentencePiece : torch::CustomClassHolder {
   // serialized model from this content_ member, thus it needs to be public.
   std::string content_;
 
-  explicit SentencePiece(const std::string& content);
-  std::vector<std::string> Encode(const std::string& input) const;
-  std::vector<int64_t> EncodeAsIds(const std::string& input) const;
-  std::string DecodeIds(const std::vector<int64_t>& ids) const;
-  std::vector<std::string> EncodeAsPieces(const std::string& input) const;
-  std::string DecodePieces(const std::vector<std::string>& pieces) const;
-  int64_t GetPieceSize() const;
-  int64_t unk_id() const;
-  int64_t PieceToId(const std::string& piece) const;
-  std::string IdToPiece(const int64_t id) const;
+  TORCHTEXT_API explicit SentencePiece(const std::string& content);
+  TORCHTEXT_API std::vector<std::string> Encode(const std::string& input) const;
+  TORCHTEXT_API std::vector<int64_t> EncodeAsIds(
+      const std::string& input) const;
+  TORCHTEXT_API std::string DecodeIds(const std::vector<int64_t>& ids) const;
+  TORCHTEXT_API std::vector<std::string> EncodeAsPieces(
+      const std::string& input) const;
+  TORCHTEXT_API std::string DecodePieces(
+      const std::vector<std::string>& pieces) const;
+  TORCHTEXT_API int64_t GetPieceSize() const;
+  TORCHTEXT_API int64_t unk_id() const;
+  TORCHTEXT_API int64_t PieceToId(const std::string& piece) const;
+  TORCHTEXT_API std::string IdToPiece(const int64_t id) const;
 };
 
 void generate_sp_model(

torchtext/csrc/vectors.h

@@ -1,4 +1,5 @@
 #include <torch/script.h>
+#include <torchtext/csrc/export.h>
 
 namespace torchtext {
 
@@ -26,22 +27,28 @@ struct Vectors : torch::CustomClassHolder {
       const IndexMap& stoi,
       torch::Tensor vectors,
       torch::Tensor unk_tensor);
-  explicit Vectors(
+  TORCHTEXT_API explicit Vectors(
       const std::vector<std::string>& tokens,
       const std::vector<std::int64_t>& indices,
       torch::Tensor vectors,
       torch::Tensor unk_tensor);
-  std::unordered_map<std::string, int64_t> get_stoi();
-  torch::Tensor __getitem__(const std::string& token);
-  torch::Tensor lookup_vectors(const std::vector<std::string>& tokens);
-  void __setitem__(const std::string& token, const torch::Tensor& vector);
-  int64_t __len__();
+  TORCHTEXT_API std::unordered_map<std::string, int64_t> get_stoi();
+  TORCHTEXT_API torch::Tensor __getitem__(const std::string& token);
+  TORCHTEXT_API torch::Tensor lookup_vectors(
+      const std::vector<std::string>& tokens);
+  TORCHTEXT_API void __setitem__(
+      const std::string& token,
+      const torch::Tensor& vector);
+  TORCHTEXT_API int64_t __len__();
 };
 
-VectorsStates _serialize_vectors(const c10::intrusive_ptr<Vectors>& self);
-c10::intrusive_ptr<Vectors> _deserialize_vectors(VectorsStates states);
+TORCHTEXT_API VectorsStates
+_serialize_vectors(const c10::intrusive_ptr<Vectors>& self);
+TORCHTEXT_API c10::intrusive_ptr<Vectors> _deserialize_vectors(
+    VectorsStates states);
 
-std::tuple<Vectors, std::vector<std::string>> _load_token_and_vectors_from_file(
+TORCHTEXT_API std::tuple<Vectors, std::vector<std::string>>
+_load_token_and_vectors_from_file(
     const std::string& file_path,
     const std::string& delimiter_str,
     const int64_t num_cpus,

torchtext/csrc/vocab.h

@@ -1,6 +1,7 @@
 #pragma once
 #include <c10/util/string_view.h>
 #include <torch/script.h>
+#include <torchtext/csrc/export.h>
 #include <algorithm>
 
 namespace torchtext {
@@ -27,7 +28,7 @@ struct CompareTokens {
   }
 };
 
-int64_t _infer_lines(const std::string& file_path);
+TORCHTEXT_API int64_t _infer_lines(const std::string& file_path);
 
 struct Vocab : torch::CustomClassHolder {
   static const int32_t MAX_VOCAB_SIZE = 30000000;
@@ -40,23 +41,24 @@ struct Vocab : torch::CustomClassHolder {
   // TODO: [can we remove this?] we need to keep this constructor, otherwise
   // torch binding gets compilation error: no matching constructor for
   // initialization of 'torchtext::Vocab'
-  explicit Vocab(StringList tokens);
-  explicit Vocab(
+  TORCHTEXT_API explicit Vocab(StringList tokens);
+  TORCHTEXT_API explicit Vocab(
       StringList tokens,
       const c10::optional<int64_t>& default_index);
-  int64_t __len__() const;
-  int64_t __getitem__(const c10::string_view& token) const;
-  bool __contains__(const c10::string_view& token) const;
-  void set_default_index(c10::optional<int64_t> index);
-  c10::optional<int64_t> get_default_index() const;
-  void insert_token(std::string token, const int64_t& index);
-  void append_token(std::string token);
-  std::string lookup_token(const int64_t& index);
-  std::vector<std::string> lookup_tokens(const std::vector<int64_t>& indices);
+  TORCHTEXT_API int64_t __len__() const;
+  TORCHTEXT_API int64_t __getitem__(const c10::string_view& token) const;
+  TORCHTEXT_API bool __contains__(const c10::string_view& token) const;
+  TORCHTEXT_API void set_default_index(c10::optional<int64_t> index);
+  TORCHTEXT_API c10::optional<int64_t> get_default_index() const;
+  TORCHTEXT_API void insert_token(std::string token, const int64_t& index);
+  TORCHTEXT_API void append_token(std::string token);
+  TORCHTEXT_API std::string lookup_token(const int64_t& index);
+  TORCHTEXT_API std::vector<std::string> lookup_tokens(
+      const std::vector<int64_t>& indices);
   std::vector<int64_t> lookup_indices(
       const std::vector<c10::string_view>& tokens);
-  std::unordered_map<std::string, int64_t> get_stoi() const;
-  std::vector<std::string> get_itos() const;
+  TORCHTEXT_API std::unordered_map<std::string, int64_t> get_stoi() const;
+  TORCHTEXT_API std::vector<std::string> get_itos() const;
 
  protected:
   uint32_t _hash(const c10::string_view& str) const {
@@ -86,14 +88,15 @@ struct Vocab : torch::CustomClassHolder {
   }
 };
 
-VocabStates _serialize_vocab(const c10::intrusive_ptr<Vocab>& self);
-c10::intrusive_ptr<Vocab> _deserialize_vocab(VocabStates states);
+TORCHTEXT_API VocabStates
+_serialize_vocab(const c10::intrusive_ptr<Vocab>& self);
+TORCHTEXT_API c10::intrusive_ptr<Vocab> _deserialize_vocab(VocabStates states);
 
-Vocab _load_vocab_from_file(
+TORCHTEXT_API Vocab _load_vocab_from_file(
     const std::string& file_path,
     const int64_t min_freq,
     const int64_t num_cpus);
-Vocab _build_vocab_from_text_file(
+TORCHTEXT_API Vocab _build_vocab_from_text_file(
     const std::string& file_path,
     const int64_t min_freq,
     const int64_t num_cpus,