Merge pull request #1299 from yueyinqiu/gradient

property grad

Author: NiklasGustafsson

RELEASENOTES.md

@@ -11,8 +11,12 @@ __Breaking Changes__:
 
 __API Changes__:
 
+- #1291 `Tensor.grad()` and `Tensor.set_grad()` have been replaced by a new property `Tensor.grad`.
+    - A potential memory leak caused by `set_grad` has been resolved.
+
 __Bug Fixes__:
 
+- #1300 `Adadelta`, `Adam` and `AdamW` will no longer throw `NullReferenceException` when `maximize` is `true` and `grad` is `null`.
 - `torch.normal` will now correctly return a leaf tensor.
 
 # NuGet Version 0.102.4

src/Examples/AdversarialExampleGeneration.cs

@@ -133,7 +133,7 @@ private static double Test(
                         model.zero_grad();
                         loss.backward();
 
-                        var perturbed = Attack(data, ε, data.grad());
+                        var perturbed = Attack(data, ε, data.grad);
 
                         using (var final = model.call(perturbed)) {
 

src/FSharp.Examples/AdversarialExampleGeneration.fs

@@ -79,7 +79,7 @@ let test (model:MNIST.Model) (eps:float) (data:Dataset) size =
             model.zero_grad()
             loss.backward()
 
-            use perturbed = attack input (eps.ToScalar()) (input.grad())
+            use perturbed = attack input (eps.ToScalar()) (input.grad)
             use final = perturbed --> model
             correct <- correct + final.argmax(1L).eq(labels).sum().ToInt32()
         end

src/TorchSharp/NN/Module.cs

@@ -239,8 +239,10 @@ private void _toEpilog(ScalarType? dtype, Device device)
                                                             .ToDictionary(field => field.ComponentName());
 
                     foreach (var (name, param) in named_parameters(false).ToList()) {
+                        using var grad = param.grad;
+
                         if (!param.toWillCopy(dtype ?? param.dtype, device ?? param.device) &&
-                            (param.grad() is null || !param.grad().toWillCopy(dtype ?? param.dtype, device ?? param.device)))
+                            (grad is null || !grad.toWillCopy(dtype ?? param.dtype, device ?? param.device)))
                             continue;
 
                         Parameter p;
@@ -252,20 +254,19 @@ private void _toEpilog(ScalarType? dtype, Device device)
                         // disable grad we would need to call .detach() on the moved tensor.
                         using (var d = torch.no_grad()) {
                             p = new Parameter(
-                                param.to(paramType, device ?? param.device).DetachFromDisposeScope(), param.requires_grad)
-                                .DetachFromDisposeScope() as Parameter;
+                                data: param.to(paramType, device ?? param.device),
+                                requires_grad: param.requires_grad);
+                            _ = p.DetachFromDisposeScope();
 
                             // Copy the gradient over as well, if it exists
-                            var grad = param.grad();
                             if (grad is not null) {
-                                p.set_grad(grad.to(paramType, device ?? param.device)
-                                                .with_requires_grad(grad.requires_grad)
-                                                .MoveToOtherDisposeScope(p));
+                                using var newGrad = grad.to(paramType, device ?? param.device)
+                                    .with_requires_grad(grad.requires_grad);
+                                p.grad = newGrad;
                             }
 
-                            // Dispose the param and gradient
+                            // Dispose the param
                             param.Dispose();
-                            grad?.Dispose();
                         }
                         ConditionallyRegisterParameter(name, p);
 
@@ -360,11 +361,10 @@ public virtual void zero_grad(bool set_to_none = true)
                     CheckForErrors();
 
                     foreach (var (_, p) in named_parameters()) {
-                        var grad = p.grad();
+                        using var grad = p.grad;
                         if (grad is not null) {
                             if (set_to_none) {
-                                p.set_grad(null);
-                                grad.DetachFromDisposeScope().Dispose();
+                                p.grad = null;
                             } else {
                                 grad.zero_();
                             }

src/TorchSharp/Optimizers/ASGD.cs

@@ -145,7 +145,7 @@ public override Tensor step(Func<Tensor> closure = null)
 
                     foreach (var param in group.Parameters) {
 
-                        var grad = param.grad();
+                        var grad = param.grad;
 
                         if (grad is null) continue;
 

src/TorchSharp/Optimizers/Adadelta.cs

@@ -136,10 +136,12 @@ public override Tensor step(Func<Tensor> closure = null)
 
                     foreach (var param in group.Parameters) {
 
-                        var grad = (maximize) ? -param.grad() : param.grad();
+                        var grad = param.grad;
 
                         if (grad is null) continue;
 
+                        if (maximize) grad = -grad;
+
                         if (grad.is_sparse) throw new ArgumentException("Adadelta does not support sparse gradients");
 
                         var state = (State)_state[param.handle];

src/TorchSharp/Optimizers/Adagrad.cs

@@ -147,7 +147,7 @@ public override Tensor step(Func<Tensor> closure = null)
 
                         var state = (State)_state[param.handle];
 
-                        var grad = param.grad();
+                        var grad = param.grad;
 
                         if (grad is null) continue;
 

src/TorchSharp/Optimizers/Adam.cs

@@ -164,10 +164,12 @@ public override Tensor step(Func<Tensor> closure = null)
 
                         var state = (State)_state[param.handle];
 
-                        var grad = (maximize) ? -param.grad() : param.grad();
+                        var grad = param.grad;
 
                         if (grad is null) continue;
 
+                        if (maximize) grad = -grad;
+
                         state.step += 1;
 
                         var bias_correction1 = 1 - Math.Pow(beta1, state.step);

src/TorchSharp/Optimizers/AdamW.cs

@@ -164,10 +164,12 @@ public override Tensor step(Func<Tensor> closure = null)
 
                         var state = (State)_state[param.handle];
 
-                        var grad = (maximize) ? -param.grad() : param.grad();
+                        var grad = param.grad;
 
                         if (grad is null) continue;
 
+                        if (maximize) grad = -grad;
+
                         state.step += 1;
 
                         param.mul_(1 - lr * weight_decay);

src/TorchSharp/Optimizers/Adamax.cs

@@ -148,7 +148,7 @@ public override Tensor step(Func<Tensor> closure = null)
 
                     foreach (var param in group.Parameters) {
 
-                        var grad = param.grad();
+                        var grad = param.grad;
 
                         if (grad is null) continue;