Merge branch 'main' into pytorch-mutex

Author: malfet

.github/workflows/build-conda-images.yml

@@ -3,7 +3,7 @@ name: Build conda docker images
 on:
   push:
     branches:
-      master
+      main
     paths:
       - conda/Dockerfile
       - 'common/*'

.github/workflows/build-libtorch-images.yml

@@ -3,7 +3,7 @@ name: Build libtorch docker images
 on:
   push:
     branches:
-      master
+      main
     paths:
       - libtorch/Dockerfile
       - 'common/*'

.github/workflows/build-llvm-images.yml

@@ -3,7 +3,7 @@ name: Build llvm docker images
 on:
   push:
     branches:
-      master
+      main
     paths:
       - llvm/*
       - .github/workflows/build-llvm-images.yml

.github/workflows/build-magma-windows.yml

@@ -3,7 +3,7 @@ name: Build MAGMA for Windows
 on:
   push:
     branches:
-      master
+      main
     paths:
       - windows/internal/build_magma.bat
   pull_request:

.github/workflows/build-manywheel-images.yml

@@ -3,7 +3,7 @@ name: Build manywheel docker images
 on:
   push:
     branches:
-      master
+      main
     paths:
       - .github/workflows/build-manywheel-images.yml
       - manywheel/Dockerfile

analytics/github_analyze.py

@@ -26,7 +26,13 @@ class GitCommit:
     author_date: datetime
     commit_date: Optional[datetime]
 
-    def __init__(self, commit_hash: str, author: str, author_date: datetime, title: str, body: str, commit_date: Optional[datetime] = None) -> None:
+    def __init__(self,
+                 commit_hash: str,
+                 author: str,
+                 author_date: datetime,
+                 title: str,
+                 body: str,
+                 commit_date: Optional[datetime] = None) -> None:
         self.commit_hash = commit_hash
         self.author = author
         self.author_date = author_date
@@ -139,9 +145,10 @@ def __init__(self, path, remote='upstream'):
         self.remote = remote
 
     def _run_git_log(self, revision_range) -> List[GitCommit]:
-        log = _check_output(["git", "-C", self.repo_dir, "log", '--format=fuller', '--date=unix', revision_range]).split("\n")
-        rc = []
-        cur_msg = []
+        log = _check_output(['git', '-C', self.repo_dir, 'log',
+                             '--format=fuller', '--date=unix', revision_range, '--', '.']).split("\n")
+        rc: List[GitCommit] = []
+        cur_msg: List[str] = []
         for line in log:
             if line.startswith("commit"):
                 if len(cur_msg) > 0:
@@ -304,30 +311,83 @@ def print_contributor_stats(commits, delta: Optional[timedelta] = None) -> None:
         print(f"{author}: {count}")
 
 
+def commits_missing_in_branch(repo: GitRepo, branch: str, orig_branch: str, milestone_idx: int) -> None:
+    def get_commits_dict(x, y):
+        return build_commit_dict(repo.get_commit_list(x, y))
+    master_commits = get_commits_dict(orig_branch, 'master')
+    release_commits = get_commits_dict(orig_branch, branch)
+    print(f"len(master_commits)={len(master_commits)}")
+    print(f"len(release_commits)={len(release_commits)}")
+    print("URL;Title;Status")
+    for issue in gh_get_milestone_issues('pytorch', 'pytorch', milestone_idx, IssueState.ALL):
+        html_url, state = issue["html_url"], issue["state"]
+        # Skip closed states if they were landed before merge date
+        if state == "closed":
+            mentioned_after_cut = any(html_url in commit_message for commit_message in master_commits.values())
+            # If issue is not mentioned after cut, that it must be present in release branch
+            if not mentioned_after_cut:
+                continue
+            mentioned_in_release = any(html_url in commit_message for commit_message in release_commits.values())
+            # if Issue is mentioned is release branch, than it was picked already
+            if mentioned_in_release:
+                continue
+        print(f'{html_url};{issue["title"]};{state}')
+
+
 def parse_arguments():
     from argparse import ArgumentParser
     parser = ArgumentParser(description="Print GitHub repo stats")
     parser.add_argument("--repo-path",
                         type=str,
                         help="Path to PyTorch git checkout",
                         default=os.path.expanduser("~/git/pytorch/pytorch"))
+    parser.add_argument("--milestone-id", type=str)
+    parser.add_argument("--branch", type=str)
+    parser.add_argument("--remote",
+                        type=str,
+                        help="Remote to base off of",
+                        default="")
     parser.add_argument("--analyze-reverts", action="store_true")
     parser.add_argument("--contributor-stats", action="store_true")
+    parser.add_argument("--missing-in-branch", action="store_true")
     return parser.parse_args()
 
 
 def main():
     import time
     args = parse_arguments()
-    remotes = get_git_remotes(args.repo_path)
-    # Pick best remote
-    remote = next(iter(remotes.keys()))
-    for key in remotes:
-        if remotes[key] == 'https://github.com/pytorch/pytorch':
-            remote = key
+    remote = args.remote
+    if not remote:
+        remotes = get_git_remotes(args.repo_path)
+        # Pick best remote
+        remote = next(iter(remotes.keys()))
+        for key in remotes:
+            if remotes[key].endswith('github.com/pytorch/pytorch'):
+                remote = key
 
     repo = GitRepo(args.repo_path, remote)
-    print("Parsing git history...", end='', flush=True)
+
+    if args.missing_in_branch:
+        # Use milestone idx or search it along milestone titles
+        try:
+            milestone_idx = int(args.milestone_id)
+        except ValueError:
+            milestone_idx = -1
+            milestones = gh_get_milestones()
+            for milestone in milestones:
+                if milestone.get('title', '') == args.milestone_id:
+                    milestone_idx = int(milestone.get('number', '-2'))
+            if milestone_idx < 0:
+                print(f'Could not find milestone {args.milestone_id}')
+                return
+
+        commits_missing_in_branch(repo,
+                                  args.branch,
+                                  f'orig/{args.branch}',
+                                  milestone_idx)
+        return
+
+    print(f"Parsing git history with remote {remote}...", end='', flush=True)
     start_time = time.time()
     x = repo._run_git_log(f"{remote}/master")
     print(f"done in {time.time()-start_time:.1f} sec")

conda/pytorch-nightly/build.sh

@@ -26,6 +26,8 @@ export INSTALL_TEST=0 # dont install test binaries into site-packages
 
 # MacOS build is simple, and will not be for CUDA
 if [[ "$OSTYPE" == "darwin"* ]]; then
+    export USE_LLVM=$CMAKE_PREFIX_PATH
+    export LLVM_DIR=$USE_LLVM/lib/cmake/llvm
     MACOSX_DEPLOYMENT_TARGET=10.9 \
         CXX=clang++ \
         CC=clang \

conda/pytorch-nightly/meta.yaml

@@ -24,6 +24,7 @@ requirements:
     - typing_extensions
     - dataclasses # [py36]
     - ninja
+    - llvmdev=9
     - libuv # [win]
     - libuv # [unix]
     - pkg-config # [unix]
@@ -73,6 +74,9 @@ build:
     - USE_WHOLE_CUDNN # [unix]
     - BUILD_TEST # [unix]
     - USE_PYTORCH_METAL_EXPORT # [osx]
+    - USE_COREML_DELEGATE # [osx]
+  features:
+{{ environ.get('CONDA_CPU_ONLY_FEATURE', '    - cpuonly # [not osx]') }}
 
 test:
  imports:

manywheel/build_rocm.sh

@@ -58,10 +58,12 @@ if [[ "$OS_NAME" == *"CentOS Linux"* ]]; then
     LIBGOMP_PATH="/usr/lib64/libgomp.so.1"
     LIBNUMA_PATH="/usr/lib64/libnuma.so.1"
     LIBELF_PATH="/usr/lib64/libelf.so.1"
+    LIBTINFO_PATH="/usr/lib64/libtinfo.so.5"
 elif [[ "$OS_NAME" == *"Ubuntu"* ]]; then
     LIBGOMP_PATH="/usr/lib/x86_64-linux-gnu/libgomp.so.1"
     LIBNUMA_PATH="/usr/lib/x86_64-linux-gnu/libnuma.so.1"
     LIBELF_PATH="/usr/lib/x86_64-linux-gnu/libelf.so.1"
+    LIBTINFO_PATH="/lib/x86_64-linux-gnu/libtinfo.so.5"
 fi
 
 # NOTE: Some ROCm versions have identical dependencies, or very close deps.
@@ -146,6 +148,7 @@ DEPS_LIST=(
     "$LIBGOMP_PATH"
     "$LIBNUMA_PATH"
     "$LIBELF_PATH"
+    "$LIBTINFO_PATH"
 )
 
 DEPS_SONAME=(
@@ -170,6 +173,7 @@ DEPS_SONAME=(
     "libgomp.so.1"
     "libnuma.so.1"
     "libelf.so.1"
+    "libtinfo.so.5"
 )
 
 DEPS_AUX_SRCLIST=(

s3_management/manage.py

@@ -98,7 +98,7 @@ def gen_file_list(
             self.nightly_packages_to_show() if self.prefix == 'whl/nightly'
             else self.objects
         )
-        subdir = self._resolve_subdir(subdir)
+        subdir = self._resolve_subdir(subdir) + '/'
         for obj in objects:
             if package_name is not None:
                 if self.obj_to_package_name(obj) != package_name: