diff --git a/THIRD-PARTY-NOTICES.TXT b/THIRD-PARTY-NOTICES.TXT index 9bdcc1090..de4414006 100644 --- a/THIRD-PARTY-NOTICES.TXT +++ b/THIRD-PARTY-NOTICES.TXT @@ -23,9 +23,9 @@ SOFTWARE. This software incorporates material from third parties listed below: ================== - + From PyTorch - + Copyright (c) 2016- Facebook, Inc (Adam Paszke) Copyright (c) 2014- Facebook, Inc (Soumith Chintala) Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) @@ -35,54 +35,54 @@ Copyright (c) 2011-2013 NYU (Clement Farabet) Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, Iain Melvin, Jason Weston) Copyright (c) 2006 Idiap Research Institute (Samy Bengio) Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, Samy Bengio, Johnny Mariethoz) - + From Caffe2: - + Copyright (c) 2016-present, Facebook Inc. All rights reserved. - + All contributions by Facebook: Copyright (c) 2016 Facebook Inc. - + All contributions by Google: Copyright (c) 2015 Google Inc. All rights reserved. - + All contributions by Yangqing Jia: Copyright (c) 2015 Yangqing Jia All rights reserved. - + All contributions from Caffe: Copyright(c) 2013, 2014, 2015, the respective contributors All rights reserved. - + All other contributions: Copyright(c) 2015, 2016 the respective contributors All rights reserved. - + Caffe2 uses a copyright model similar to Caffe: each contributor holds copyright over their contributions to Caffe2. The project versioning records all such contribution and copyright details. If a contributor wants to further mark their specific copyright on a particular contribution, they should indicate their copyright solely in the commit message of the change when it is committed. - + All rights reserved. - + Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - + 3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories America and IDIAP Research Institute nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. - + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE @@ -591,4 +591,62 @@ Apache License DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + ================== + Android Code(gif_encoder) + https://cs.android.com/android/platform/superproject/+/master:external/glide/LICENSE + https://cs.android.com/android/platform/superproject/+/master:external/glide/third_party/gif_encoder/LICENSE + + Copyright 2014 Google, Inc. All rights reserved. + + Redistribution and use in source and binary forms, with or without modification, are + permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, this list + of conditions and the following disclaimer in the documentation and/or other materials + provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY GOOGLE, INC. ``AS IS'' AND ANY EXPRESS OR IMPLIED + WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GOOGLE, INC. OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + The views and conclusions contained in the software and documentation are those of the + authors and should not be interpreted as representing official policies, either expressed + or implied, of Google, Inc. + --------------------------------------------------------------------------------------------- + + License for Encoder.cs and LZWEncoder.cs + + No copyright asserted on the source code of this class. May be used for any + purpose, however, refer to the Unisys LZW patent for restrictions on use of + the associated LZWEncoder class. Please forward any corrections to + kweiner@fmsware.com. + + ----------------------------------------------------------------------------- + License for NeuQuant.cs + + Copyright (c) 1994 Anthony Dekker + + NEUQUANT Neural-Net quantization algorithm by Anthony Dekker, 1994. See + "Kohonen neural networks for optimal colour quantization" in "Network: + Computation in Neural Systems" Vol. 5 (1994) pp 351-367. for a discussion of + the algorithm. + + Any party obtaining a copy of these files from the author, directly or + indirectly, is granted, free of charge, a full and unrestricted irrevocable, + world-wide, paid up, royalty-free, nonexclusive right and license to deal in + this software and documentation files (the "Software"), including without + limitation the rights to use, copy, modify, merge, publish, distribute, + sublicense, and/or sell copies of the Software, and to permit persons who + receive copies from any such party to do so, with the only requirement being + that this copyright notice remain intact. \ No newline at end of file diff --git a/src/Examples/Examples.csproj b/src/Examples/Examples.csproj index c0ebb534c..4bac9a142 100644 --- a/src/Examples/Examples.csproj +++ b/src/Examples/Examples.csproj @@ -24,6 +24,7 @@ + diff --git a/src/Examples/Program.cs b/src/Examples/Program.cs index d66cd2712..6e6df9d58 100644 --- a/src/Examples/Program.cs +++ b/src/Examples/Program.cs @@ -14,6 +14,7 @@ public static void Main(string[] args) //ImageTransforms.Main(args); //SpeechCommands.Main(args); IOReadWrite.Main(args); + Tensorboard.Main(args).Wait(); } } } diff --git a/src/Examples/Tensorboard.cs b/src/Examples/Tensorboard.cs new file mode 100644 index 000000000..b1736b10a --- /dev/null +++ b/src/Examples/Tensorboard.cs @@ -0,0 +1,114 @@ +// Copyright (c) .NET Foundation and Contributors. All Rights Reserved. See LICENSE in the project root for license information. +using System; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Threading.Tasks; +using SixLabors.ImageSharp; +using SixLabors.ImageSharp.PixelFormats; +using SkiaSharp; +using static TorchSharp.torch; + +namespace TorchSharp.Examples +{ + public static class Tensorboard + { + private static string imagePath = "TensorboardExample/Images"; + internal static async Task Main(string[] args) + { + await DownloadExampleData("https://agirls.aottercdn.com/media/0d532b3f-0196-466a-96e1-7c6db56d0142.gif"); + var device = cuda.is_available() ? CUDA : CPU; + var writer = utils.tensorboard.SummaryWriter("tensorboard"); + Console.WriteLine($"Running Tensorboard on {device.type}"); + AddText(writer); + AddImageUsePath(writer); + AddImageUseTensor(writer, device); + AddVideo(writer, device); + AddHistogram(writer, device); + } + + private static void AddHistogram(Modules.SummaryWriter writer, Device device) + { + for (int i = 0; i < 10; i++) { + Tensor x = randn(1000, device: device); + writer.add_histogram("AddHistogram", x + i, i); + } + } + + private static void AddText(Modules.SummaryWriter writer) + { + writer.add_text("AddText", "step_1", 1); + writer.add_text("AddText", "step_2", 2); + writer.add_text("AddText", "step_3", 3); + writer.add_text("AddText", "step_4", 4); + writer.add_text("AddText", "step_5", 5); + } + + private static void AddImageUsePath(Modules.SummaryWriter writer) + { + var imagesPath = Directory.GetFiles(imagePath); + for (var i = 0; i < imagesPath.Length; i++) { + writer.add_img("AddImageUsePath", imagesPath[i], i); + } + } + + private static void AddImageUseTensor(Modules.SummaryWriter writer, Device device) + { + var images = Directory.GetFiles(imagePath).Select(item => SKBitmap.Decode(item)).ToArray(); + using var d = NewDisposeScope(); + for (var i = 0; i < images.Length; i++) { + var tensor = SKBitmapToTensor(images[i], device); + writer.add_img("AddImageUseTensor", tensor, i, dataformats: "CHW"); + images[i].Dispose(); + } + } + + private static void AddVideo(Modules.SummaryWriter writer, Device device) + { + var images = Directory.GetFiles(imagePath).Select(item => SKBitmap.Decode(item)).ToArray(); + using var d = NewDisposeScope(); + var tensor = stack(images.Select(item => SKBitmapToTensor(item, device)).ToArray()); + tensor = stack(new Tensor[] { tensor, tensor, tensor, tensor, tensor, tensor, tensor, tensor, tensor, tensor }); + foreach (var image in images) + image.Dispose(); + writer.add_video("AddVideo", tensor, 1, 10); + writer.add_video("AddVideo", tensor, 2, 10); + writer.add_video("AddVideo", tensor, 3, 10); + } + + private static Tensor SKBitmapToTensor(SKBitmap skBitmap, Device device) + { + var width = skBitmap.Width; + var height = skBitmap.Height; + var hwcData = new byte[4, height, width]; + for (var y = 0; y < height; y++) { + for (var x = 0; x < width; x++) { + var color = skBitmap.GetPixel(x, y); + hwcData[0, y, x] = color.Red; + hwcData[1, y, x] = color.Green; + hwcData[2, y, x] = color.Blue; + hwcData[3, y, x] = color.Alpha; + } + } + + return tensor(hwcData, ScalarType.Byte, device); + } + + private static async Task DownloadExampleData(string url) + { + if (Directory.Exists(imagePath)) + Directory.Delete(imagePath, true); + Directory.CreateDirectory(imagePath); + using var client = new HttpClient(); + + using var message = await client.GetAsync(url); + using var stream = await message.Content.ReadAsStreamAsync(); + using var animatedImage = Image.Load(stream); + for (var i = 0; i < animatedImage.Frames.Count; i++) { + var pngFilename = Path.Combine(imagePath, $"frame_{i}.png"); + using var pngImage = animatedImage.Frames.CloneFrame(i); + pngImage.SaveAsPng(pngFilename); + } + } + } +} diff --git a/src/TorchSharp/Tensor/Enums/HistogramBinSelector.cs b/src/TorchSharp/Tensor/Enums/HistogramBinSelector.cs index 2b085edb3..292e09443 100644 --- a/src/TorchSharp/Tensor/Enums/HistogramBinSelector.cs +++ b/src/TorchSharp/Tensor/Enums/HistogramBinSelector.cs @@ -2,9 +2,9 @@ namespace TorchSharp { - public enum HistogramBinSelector + public enum HistogramBinSelector : byte { - Doane, + Doane = 0, Rice, Scott, Sqrt, diff --git a/src/TorchSharp/Tensor/Enums/side.cs b/src/TorchSharp/Tensor/Enums/side.cs deleted file mode 100644 index 4603f68d3..000000000 --- a/src/TorchSharp/Tensor/Enums/side.cs +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright (c) .NET Foundation and Contributors. All Rights Reserved. See LICENSE in the project root for license information. -namespace TorchSharp -{ - public enum side - { - left, - right - } -} \ No newline at end of file diff --git a/src/TorchSharp/Tensor/torch.OtherOperations.cs b/src/TorchSharp/Tensor/torch.OtherOperations.cs index 0535e16b7..c6d0a1fec 100644 --- a/src/TorchSharp/Tensor/torch.OtherOperations.cs +++ b/src/TorchSharp/Tensor/torch.OtherOperations.cs @@ -628,17 +628,6 @@ public static Tensor[] meshgrid(IEnumerable tensors, string indexing = " /// public static Tensor roll(Tensor input, ReadOnlySpan shifts, ReadOnlySpan dims = default) => input.roll(shifts, dims); - // https://pytorch.org/docs/stable/generated/torch.searchsorted - [Obsolete("not implemented", true)] - static Tensor searchsorted( - Tensor sorted_sequence, - Tensor values, - bool out_int32 = false, - bool right = false, - side side = side.left, - Tensor? sorter = null) - => throw new NotImplementedException(); - // https://pytorch.org/docs/stable/generated/torch.tensordot /// /// Returns a contraction of and over multiple dimensions. diff --git a/src/TorchSharp/TorchSharp.csproj b/src/TorchSharp/TorchSharp.csproj index 7fe2e227a..b17d8b1ba 100644 --- a/src/TorchSharp/TorchSharp.csproj +++ b/src/TorchSharp/TorchSharp.csproj @@ -30,6 +30,7 @@ + diff --git a/src/TorchSharp/Utils/tensorboard/Enums/HistogramBinSelector.cs b/src/TorchSharp/Utils/tensorboard/Enums/HistogramBinSelector.cs new file mode 100644 index 000000000..1917bda87 --- /dev/null +++ b/src/TorchSharp/Utils/tensorboard/Enums/HistogramBinSelector.cs @@ -0,0 +1,13 @@ +namespace TorchSharp.Utils.tensorboard.Enums +{ + public enum HistogramBinSelector : byte + { + Doane = 0, + Rice, + Scott, + Sqrt, + Stone, + Sturges, + Tensorflow + } +} diff --git a/src/TorchSharp/Utils/tensorboard/GifEncoder/Encoder.cs b/src/TorchSharp/Utils/tensorboard/GifEncoder/Encoder.cs new file mode 100644 index 000000000..328427b1a --- /dev/null +++ b/src/TorchSharp/Utils/tensorboard/GifEncoder/Encoder.cs @@ -0,0 +1,519 @@ +using System; +using System.IO; +using SkiaSharp; + +namespace TorchSharp +{ + public static partial class torch + { + public static partial class utils + { + public static partial class tensorboard + { + internal static partial class GifEncoder + { + /// + /// Class AnimatedGifEncoder - Encodes a GIF file consisting of one or more frames. + /// + /// No copyright asserted on the source code of this class. May be used for any + /// purpose, however, refer to the Unisys LZW patent for restrictions on use of + /// the associated LZWEncoder class. Please forward any corrections to + /// kweiner@fmsware.com. + /// + /// @author Kevin Weiner, FM Software + /// @version 1.03 November 2003 + /// + /// https://cs.android.com/android/platform/superproject/+/master:external/glide/third_party/gif_encoder/src/main/java/com/bumptech/glide/gifencoder/AnimatedGifEncoder.java + /// + internal class Encoder : IDisposable + { + protected int width; // image size + protected int height; + protected SKColor transparent = SKColor.Empty; // transparent color if given + protected int transIndex; // transparent index in color table + protected int repeat = -1; // no repeat + protected int delay = 0; // frame delay (hundredths) + protected bool started = false; // ready to output frames + // protected BinaryWriter bw; + protected MemoryStream ms; + // protected FileStream fs; + + protected SKBitmap image; // current frame + protected byte[] pixels; // BGR byte array from frame + protected byte[] indexedPixels; // converted frame indexed to palette + protected int colorDepth; // number of bit planes + protected byte[] colorTab; // RGB palette + protected bool[] usedEntry = new bool[256]; // active palette entries + protected int palSize = 7; // color table size (bits-1) + protected int dispose = -1; // disposal code (-1 = use default) + protected bool closeStream = false; // close stream when finished + protected bool firstFrame = true; + protected bool sizeSet = false; // if false, get size from first frame + protected int sample = 10; // default sample interval for quantizer + private bool disposedValue; + + /// + /// Sets the delay time between each frame, or changes it + /// for subsequent frames (applies to last frame added). + /// + /// delay time in milliseconds + public void SetDelay(int ms) + { + delay = (int)Math.Round(ms / 10.0f); + } + + /// + /// Sets the GIF frame disposal code for the last added frame + /// and any subsequent frames. Default is 0 if no transparent + /// color has been set, otherwise 2. + /// + /// disposal code. + public void SetDispose(int code) + { + if (code >= 0) { + dispose = code; + } + } + + /// + /// Sets the number of times the set of GIF frames + /// should be played. Default is 1; 0 means play + /// indefinitely. Must be invoked before the first + /// image is added. + /// + /// number of iterations. + public void SetRepeat(int iter) + { + if (iter >= 0) { + repeat = iter; + } + } + + /// + /// Sets the transparent color for the last added frame + /// and any subsequent frames. + /// Since all colors are subject to modification + /// in the quantization process, the color in the final + /// palette for each frame closest to the given color + /// becomes the transparent color for that frame. + /// May be set to null to indicate no transparent color. + /// + /// Color to be treated as transparent on display. + public void SetTransparent(SKColor c) + { + transparent = c; + } + + /// + /// Adds next GIF frame. The frame is not written immediately, but is + /// actually deferred until the next frame is received so that timing + /// data can be inserted. Invoking finish() flushes all + /// frames. If setSize was not invoked, the size of the + /// first image is used for all subsequent frames. + /// + /// BufferedImage containing frame to write. + /// true if successful. + public bool AddFrame(SKBitmap im) + { + if ((im == null) || !started) { + return false; + } + bool ok = true; + try { + if (!sizeSet) { + // use first frame's size + SetSize(im.Width, im.Height); + } + image = im; + GetImagePixels(); // convert to correct format if necessary + AnalyzePixels(); // build color table & map pixels + if (firstFrame) { + WriteLSD(); // logical screen descriptior + WritePalette(); // global color table + if (repeat >= 0) { + // use NS app extension to indicate reps + WriteNetscapeExt(); + } + } + WriteGraphicCtrlExt(); // write graphic control extension + WriteImageDesc(); // image descriptor + if (!firstFrame) { + WritePalette(); // local color table + } + WritePixels(); // encode and write pixel data + firstFrame = false; + } catch (IOException) { + ok = false; + } + + return ok; + } + + /// + /// Flushes any pending data and closes output file. + /// If writing to an OutputStream, the stream is not + /// closed. + /// + /// + public bool Finish() + { + if (!started) return false; + bool ok = true; + started = false; + try { + ms.WriteByte(0x3b); // gif trailer + ms.Flush(); + } catch (IOException) { + ok = false; + } + + // reset for subsequent use + transIndex = 0; + // fs = null; + image = null; + pixels = null; + indexedPixels = null; + colorTab = null; + closeStream = false; + firstFrame = true; + + return ok; + } + + /// + /// Sets frame rate in frames per second. Equivalent to + /// setDelay(1000/fps). + /// + /// fps float frame rate (frames per second) + public void SetFrameRate(float fps) + { + if (fps != 0f) { + delay = (int)Math.Round(100f / fps); + } + } + + /// + /// Sets the GIF frame size. The default size is the + /// size of the first frame added if this method is + /// not invoked. + /// + /// frame width + /// frame width + public void SetSize(int w, int h) + { + if (started && !firstFrame) return; + width = w; + height = h; + if (width < 1) width = 320; + if (height < 1) height = 240; + sizeSet = true; + } + + /// + /// Initiates GIF file creation on the given stream. The stream + /// is not closed automatically. + /// + /// OutputStream on which GIF images are written. + /// false if initial write failed. + public bool Start(MemoryStream os) + { + if (os == null) return false; + bool ok = true; + closeStream = false; + ms = os; + try { + WriteString("GIF89a"); // header + } catch (IOException) { + ok = false; + } + return started = ok; + } + + /// + /// Initiates writing of a GIF file to a memory stream. + /// + /// + public bool Start() + { + bool ok; + try { + ok = Start(new MemoryStream(10 * 1024)); + closeStream = true; + } catch (IOException) { + ok = false; + } + return started = ok; + } + + /// + /// Initiates writing of a GIF file with the specified name. + /// + /// + /// + public bool Output(string file) + { + try { + var fs = new FileStream(file, FileMode.OpenOrCreate, FileAccess.Write, FileShare.None); + fs.Write(ms.ToArray(), 0, (int)ms.Length); + fs.Close(); + } catch (IOException) { + return false; + } + return true; + } + + public MemoryStream Output() + { + return ms; + } + + /// + /// Analyzes image colors and creates color map. + /// + protected void AnalyzePixels() + { + int len = pixels.Length; + int nPix = len / 3; + indexedPixels = new byte[nPix]; + var nq = new NeuQuant(pixels, len, sample); + // initialize quantizer + colorTab = nq.Process(); // create reduced palette + // convert map from BGR to RGB + // for (int i = 0; i < colorTab.Length; i += 3) + // { + // byte temp = colorTab[i]; + // colorTab[i] = colorTab[i + 2]; + // colorTab[i + 2] = temp; + // usedEntry[i / 3] = false; + // } + // map image pixels to new palette + int k = 0; + for (int i = 0; i < nPix; i++) { + int index = + nq.Map(pixels[k++] & 0xff, + pixels[k++] & 0xff, + pixels[k++] & 0xff); + usedEntry[index] = true; + indexedPixels[i] = (byte)index; + } + pixels = null; + colorDepth = 8; + palSize = 7; + // get closest match to transparent color if specified + if (transparent != SKColor.Empty) { + //transIndex = FindClosest(transparent); + transIndex = nq.Map(transparent.Blue, transparent.Green, transparent.Red); + } + } + + /// + /// Returns index of palette color closest to c + /// + /// + /// + protected int FindClosest(SKColor c) + { + if (colorTab == null) return -1; + int r = c.Red; + int g = c.Green; + int b = c.Blue; + int minpos = 0; + int dmin = 256 * 256 * 256; + int len = colorTab.Length; + for (int i = 0; i < len;) { + int dr = r - (colorTab[i++] & 0xff); + int dg = g - (colorTab[i++] & 0xff); + int db = b - (colorTab[i] & 0xff); + int d = dr * dr + dg * dg + db * db; + int index = i / 3; + if (usedEntry[index] && (d < dmin)) { + dmin = d; + minpos = index; + } + i++; + } + return minpos; + } + + /// + /// Extracts image pixels into byte array "pixels" + /// + protected void GetImagePixels() + { + pixels = new byte[3 * image.Width * image.Height]; + int count = 0; + + for (int th = 0; th < image.Height; th++) { + for (int tw = 0; tw < image.Width; tw++) { + SKColor color = image.GetPixel(tw, th); + pixels[count] = color.Red; + count++; + pixels[count] = color.Green; + count++; + pixels[count] = color.Blue; + count++; + } + } + + // pixels = ((DataBufferByte) image.getRaster().getDataBuffer()).getData(); + } + + /// + /// Writes Graphic Control Extension + /// + protected void WriteGraphicCtrlExt() + { + ms.WriteByte(0x21); // extension introducer + ms.WriteByte(0xf9); // GCE label + ms.WriteByte(4); // data block size + int transp, disp; + if (transparent == SKColor.Empty) { + transp = 0; + disp = 0; // dispose = no action + } else { + transp = 1; + disp = 2; // force clear if using transparent color + } + if (dispose >= 0) { + disp = dispose & 7; // user override + } + disp <<= 2; + + // packed fields + ms.WriteByte(Convert.ToByte(0 | // 1:3 reserved + disp | // 4:6 disposal + 0 | // 7 user input - 0 = none + transp)); // 8 transparency flag + + WriteShort(delay); // delay x 1/100 sec + ms.WriteByte(Convert.ToByte(transIndex)); // transparent color index + ms.WriteByte(0); // block terminator + } + + /// + /// Writes Image Descriptor + /// + protected void WriteImageDesc() + { + ms.WriteByte(0x2c); // image separator + WriteShort(0); // image position x,y = 0,0 + WriteShort(0); + WriteShort(width); // image size + WriteShort(height); + // packed fields + if (firstFrame) { + // no LCT - GCT is used for first (or only) frame + ms.WriteByte(0); + } else { + // specify normal LCT + ms.WriteByte(Convert.ToByte(0x80 | // 1 local color table 1=yes + 0 | // 2 interlace - 0=no + 0 | // 3 sorted - 0=no + 0 | // 4-5 reserved + palSize)); // 6-8 size of color table + } + } + + /// + /// Writes Logical Screen Descriptor + /// + protected void WriteLSD() + { + // logical screen size + WriteShort(width); + WriteShort(height); + // packed fields + ms.WriteByte(Convert.ToByte(0x80 | // 1 : global color table flag = 1 (gct used) + 0x70 | // 2-4 : color resolution = 7 + 0x00 | // 5 : gct sort flag = 0 + palSize)); // 6-8 : gct size + + ms.WriteByte(0); // background color index + ms.WriteByte(0); // pixel aspect ratio - assume 1:1 + } + + /// + /// Writes Netscape application extension to define repeat count. + /// + protected void WriteNetscapeExt() + { + ms.WriteByte(0x21); // extension introducer + ms.WriteByte(0xff); // app extension label + ms.WriteByte(11); // block size + WriteString("NETSCAPE" + "2.0"); // app id + auth code + ms.WriteByte(3); // sub-block size + ms.WriteByte(1); // loop sub-block id + WriteShort(repeat); // loop count (extra iterations, 0=repeat forever) + ms.WriteByte(0); // block terminator + } + + /// + /// Writes color table + /// + protected void WritePalette() + { + ms.Write(colorTab, 0, colorTab.Length); + int n = (3 * 256) - colorTab.Length; + for (int i = 0; i < n; i++) { + ms.WriteByte(0); + } + } + + /// + /// Encodes and writes pixel data + /// + protected void WritePixels() + { + var encoder = new LZWEncoder(indexedPixels, colorDepth); + encoder.Encode(ms); + } + + /// + /// Write 16-bit value to output stream, LSB first + /// + /// + protected void WriteShort(int value) + { + ms.WriteByte(Convert.ToByte(value & 0xff)); + ms.WriteByte(Convert.ToByte((value >> 8) & 0xff)); + } + + /// + /// Writes string to output stream + /// + /// + protected void WriteString(string s) + { + char[] chars = s.ToCharArray(); + for (int i = 0; i < chars.Length; i++) { + ms.WriteByte((byte)chars[i]); + } + } + + protected virtual void Dispose(bool disposing) + { + if (!disposedValue) { + if (disposing) { + ms.Dispose(); + } + + disposedValue = true; + } + } + + ~Encoder() + { + Dispose(disposing: false); + } + + public void Dispose() + { + Dispose(disposing: true); + GC.SuppressFinalize(this); + } + } + } + } + } + } +} diff --git a/src/TorchSharp/Utils/tensorboard/GifEncoder/LZWEncoder.cs b/src/TorchSharp/Utils/tensorboard/GifEncoder/LZWEncoder.cs new file mode 100644 index 000000000..8ad282d01 --- /dev/null +++ b/src/TorchSharp/Utils/tensorboard/GifEncoder/LZWEncoder.cs @@ -0,0 +1,333 @@ +using System; +using System.IO; + +namespace TorchSharp +{ + public static partial class torch + { + public static partial class utils + { + public static partial class tensorboard + { + internal static partial class GifEncoder + { + /// + /// Adapted from Jef Poskanzer's Java port by way of J. M. G. Elliott. + /// K Weiner 12/00 + /// + /// https://cs.android.com/android/platform/superproject/+/master:external/glide/third_party/gif_encoder/src/main/java/com/bumptech/glide/gifencoder/LZWEncoder.java + /// + internal class LZWEncoder + { + private static readonly int EOF = -1; + + private byte[] pixAry; + private int initCodeSize; + private int curPixel; + + // GIFCOMPR.C - GIF Image compression routines + // + // Lempel-Ziv compression based on 'compress'. GIF modifications by + // David Rowley (mgardi@watdcsu.waterloo.edu) + + // General DEFINEs + + static readonly int BITS = 12; + + static readonly int HSIZE = 5003; // 80% occupancy + + // GIF Image compression - modified 'compress' + // + // Based on: compress.c - File compression ala IEEE Computer, June 1984. + // + // By Authors: Spencer W. Thomas (decvax!harpo!utah-cs!utah-gr!thomas) + // Jim McKie (decvax!mcvax!jim) + // Steve Davies (decvax!vax135!petsd!peora!srd) + // Ken Turkowski (decvax!decwrl!turtlevax!ken) + // James A. Woods (decvax!ihnp4!ames!jaw) + // Joe Orost (decvax!vax135!petsd!joe) + + int n_bits; // number of bits/code + int maxbits = BITS; // user settable max # bits/code + int maxcode; // maximum code, given n_bits + int maxmaxcode = 1 << BITS; // should NEVER generate this code + + int[] htab = new int[HSIZE]; + int[] codetab = new int[HSIZE]; + + int hsize = HSIZE; // for dynamic table sizing + + int free_ent = 0; // first unused entry + + // block compression parameters -- after all codes are used up, + // and compression rate changes, start over. + bool clear_flg = false; + + // Algorithm: use open addressing double hashing (no chaining) on the + // prefix code / next character combination. We do a variant of Knuth's + // algorithm D (vol. 3, sec. 6.4) along with G. Knott's relatively-prime + // secondary probe. Here, the modular division first probe is gives way + // to a faster exclusive-or manipulation. Also do block compression with + // an adaptive reset, whereby the code table is cleared when the compression + // ratio decreases, but after the table fills. The variable-length output + // codes are re-sized at this point, and a special CLEAR code is generated + // for the decompressor. Late addition: construct the table according to + // file size for noticeable speed improvement on small files. Please direct + // questions about this implementation to ames!jaw. + + int g_init_bits; + + int ClearCode; + int EOFCode; + + // output + // + // Output the given code. + // Inputs: + // code: A n_bits-bit integer. If == -1, then EOF. This assumes + // that n_bits =< wordsize - 1. + // Outputs: + // Outputs code to the file. + // Assumptions: + // Chars are 8 bits long. + // Algorithm: + // Maintain a BITS character long buffer (so that 8 codes will + // fit in it exactly). Use the VAX insv instruction to insert each + // code in turn. When the buffer fills up empty it and start over. + + int cur_accum = 0; + int cur_bits = 0; + + int[] masks = + { + 0x0000, + 0x0001, + 0x0003, + 0x0007, + 0x000F, + 0x001F, + 0x003F, + 0x007F, + 0x00FF, + 0x01FF, + 0x03FF, + 0x07FF, + 0x0FFF, + 0x1FFF, + 0x3FFF, + 0x7FFF, + 0xFFFF + }; + + /// + /// Number of characters so far in this 'packet' + /// + int a_count; + + /// + /// Define the storage for the packet accumulator + /// + byte[] accum = new byte[256]; + + public LZWEncoder(byte[] pixels, int color_depth) + { + pixAry = pixels; + initCodeSize = Math.Max(2, color_depth); + } + + /// + /// Add a character to the end of the current packet, and if it is 254 + /// characters, flush the packet to disk. + /// + /// + /// + void Add(byte c, Stream outs) + { + accum[a_count++] = c; + if (a_count >= 254) + Flush(outs); + } + + /// + /// table clear for block compress + /// + /// + void ClearTable(Stream outs) + { + ResetCodeTable(hsize); + free_ent = ClearCode + 2; + clear_flg = true; + + Output(ClearCode, outs); + } + + /// + /// reset code table + /// + /// + void ResetCodeTable(int hsize) + { + for (int i = 0; i < hsize; ++i) + htab[i] = -1; + } + + void Compress(int init_bits, Stream outs) + { + int fcode; + int i /* = 0 */; + int c; + int ent; + int disp; + int hsize_reg; + int hshift; + + // Set up the globals: g_init_bits - initial number of bits + g_init_bits = init_bits; + + // Set up the necessary values + clear_flg = false; + n_bits = g_init_bits; + maxcode = MaxCode(n_bits); + + ClearCode = 1 << (init_bits - 1); + EOFCode = ClearCode + 1; + free_ent = ClearCode + 2; + + a_count = 0; // clear packet + + ent = NextPixel(); + + hshift = 0; + for (fcode = hsize; fcode < 65536; fcode *= 2) + ++hshift; + hshift = 8 - hshift; // set hash code range bound + + hsize_reg = hsize; + ResetCodeTable(hsize_reg); // clear hash table + + Output(ClearCode, outs); + + outer_loop: + while ((c = NextPixel()) != EOF) { + fcode = (c << maxbits) + ent; + i = (c << hshift) ^ ent; // xor hashing + + if (htab[i] == fcode) { + ent = codetab[i]; + continue; + } else if (htab[i] >= 0) // non-empty slot + { + disp = hsize_reg - i; // secondary hash (after G. Knott) + if (i == 0) + disp = 1; + do { + if ((i -= disp) < 0) + i += hsize_reg; + + if (htab[i] == fcode) { + ent = codetab[i]; + goto outer_loop; + } + } while (htab[i] >= 0); + } + Output(ent, outs); + ent = c; + if (free_ent < maxmaxcode) { + codetab[i] = free_ent++; // code -> hashtable + htab[i] = fcode; + } else + ClearTable(outs); + } + // Put out the final code. + Output(ent, outs); + Output(EOFCode, outs); + } + + public void Encode(Stream os) + { + os.WriteByte(Convert.ToByte(initCodeSize)); // write "initial code size" byte + curPixel = 0; + + Compress(initCodeSize + 1, os); // compress and write the pixel data + + os.WriteByte(0); // write block terminator + } + + /// + /// Flush the packet to disk, and reset the accumulator + /// + /// + void Flush(Stream outs) + { + if (a_count > 0) { + outs.WriteByte(Convert.ToByte(a_count)); + outs.Write(accum, 0, a_count); + a_count = 0; + } + } + + int MaxCode(int n_bits) + { + return (1 << n_bits) - 1; + } + + /// + /// Return the next pixel from the image + /// + /// + private int NextPixel() + { + int upperBound = pixAry.GetUpperBound(0); + + return (curPixel <= upperBound) ? (pixAry[curPixel++] & 0xff) : EOF; + } + + void Output(int code, Stream outs) + { + cur_accum &= masks[cur_bits]; + + if (cur_bits > 0) + cur_accum |= (code << cur_bits); + else + cur_accum = code; + + cur_bits += n_bits; + + while (cur_bits >= 8) { + Add((byte)(cur_accum & 0xff), outs); + cur_accum >>= 8; + cur_bits -= 8; + } + + // If the next entry is going to be too big for the code size, + // then increase it, if possible. + if (free_ent > maxcode || clear_flg) { + if (clear_flg) { + maxcode = MaxCode(n_bits = g_init_bits); + clear_flg = false; + } else { + ++n_bits; + if (n_bits == maxbits) + maxcode = maxmaxcode; + else + maxcode = MaxCode(n_bits); + } + } + + if (code == EOFCode) { + // At EOF, write the rest of the buffer. + while (cur_bits > 0) { + Add((byte)(cur_accum & 0xff), outs); + cur_accum >>= 8; + cur_bits -= 8; + } + + Flush(outs); + } + } + } + } + } + } + } +} diff --git a/src/TorchSharp/Utils/tensorboard/GifEncoder/NeuQuant.cs b/src/TorchSharp/Utils/tensorboard/GifEncoder/NeuQuant.cs new file mode 100644 index 000000000..be7fbcd7b --- /dev/null +++ b/src/TorchSharp/Utils/tensorboard/GifEncoder/NeuQuant.cs @@ -0,0 +1,509 @@ +using System; + +namespace TorchSharp +{ + public static partial class torch + { + public static partial class utils + { + public static partial class tensorboard + { + internal static partial class GifEncoder + { + /// + /// NeuQuant Neural-Net Quantization Algorithm + /// ------------------------------------------ + /// + /// Copyright (c) 1994 Anthony Dekker + /// + /// NEUQUANT Neural-Net quantization algorithm by Anthony Dekker, 1994. See + /// "Kohonen neural networks for optimal colour quantization" in "Network: + /// Computation in Neural Systems" Vol. 5 (1994) pp 351-367. for a discussion of + /// the algorithm. + /// + /// Any party obtaining a copy of these files from the author, directly or + /// indirectly, is granted, free of charge, a full and unrestricted irrevocable, + /// world-wide, paid up, royalty-free, nonexclusive right and license to deal in + /// this software and documentation files (the "Software"), including without + /// limitation the rights to use, copy, modify, merge, publish, distribute, + /// sublicense, and/or sell copies of the Software, and to permit persons who + /// receive copies from any such party to do so, with the only requirement being + /// that this copyright notice remain intact. + /// + /// https://cs.android.com/android/platform/superproject/+/master:external/glide/third_party/gif_encoder/src/main/java/com/bumptech/glide/gifencoder/NeuQuant.java + /// + internal class NeuQuant + { + protected static readonly int netsize = 256; /* number of colours used */ + /* four primes near 500 - assume no image has a length so large */ + /* that it is divisible by all four primes */ + protected static readonly int prime1 = 499; + protected static readonly int prime2 = 491; + protected static readonly int prime3 = 487; + protected static readonly int prime4 = 503; + protected static readonly int minpicturebytes = (3 * prime4); + /* minimum size for input image */ + /* Program Skeleton + ---------------- + [select samplefac in range 1..30] + [read image from input file] + pic = (unsigned char*) malloc(3*width*height); + initnet(pic,3*width*height,samplefac); + learn(); + unbiasnet(); + [write output image header, using writecolourmap(f)] + inxbuild(); + write output image using inxsearch(b,g,r) */ + + /* Network Definitions + ------------------- */ + protected static readonly int maxnetpos = (netsize - 1); + protected static readonly int netbiasshift = 4; /* bias for colour values */ + protected static readonly int ncycles = 100; /* no. of learning cycles */ + + /* defs for freq and bias */ + protected static readonly int intbiasshift = 16; /* bias for fractions */ + protected static readonly int intbias = (((int)1) << intbiasshift); + protected static readonly int gammashift = 10; /* gamma = 1024 */ + protected static readonly int gamma = (((int)1) << gammashift); + protected static readonly int betashift = 10; + protected static readonly int beta = (intbias >> betashift); /* beta = 1/1024 */ + protected static readonly int betagamma = + (intbias << (gammashift - betashift)); + + /* defs for decreasing radius factor */ + protected static readonly int initrad = (netsize >> 3); /* for 256 cols, radius starts */ + protected static readonly int radiusbiasshift = 6; /* at 32.0 biased by 6 bits */ + protected static readonly int radiusbias = (((int)1) << radiusbiasshift); + protected static readonly int initradius = (initrad * radiusbias); /* and decreases by a */ + protected static readonly int radiusdec = 30; /* factor of 1/30 each cycle */ + + /* defs for decreasing alpha factor */ + protected static readonly int alphabiasshift = 10; /* alpha starts at 1.0 */ + protected static readonly int initalpha = (((int)1) << alphabiasshift); + + protected int alphadec; /* biased by 10 bits */ + + /* radbias and alpharadbias used for radpower calculation */ + protected static readonly int radbiasshift = 8; + protected static readonly int radbias = (((int)1) << radbiasshift); + protected static readonly int alpharadbshift = (alphabiasshift + radbiasshift); + protected static readonly int alpharadbias = (((int)1) << alpharadbshift); + + /* Types and Global Variables + -------------------------- */ + + protected byte[] thepicture; /* the input image itself */ + protected int lengthcount; /* lengthcount = H*W*3 */ + + protected int samplefac; /* sampling factor 1..30 */ + + // typedef int pixel[4]; /* BGRc */ + protected int[][] network; /* the network itself - [netsize][4] */ + + protected int[] netindex = new int[256]; + /* for network lookup - really 256 */ + + protected int[] bias = new int[netsize]; + /* bias and freq arrays for learning */ + protected int[] freq = new int[netsize]; + protected int[] radpower = new int[initrad]; + /* radpower for precomputation */ + + /// + /// Initialise network in range (0,0,0) to (255,255,255) and set parameters + /// + /// + /// + /// + public NeuQuant(byte[] thepic, int len, int sample) + { + + int i; + int[] p; + + thepicture = thepic; + lengthcount = len; + samplefac = sample; + + network = new int[netsize][]; + for (i = 0; i < netsize; i++) { + network[i] = new int[4]; + p = network[i]; + p[0] = p[1] = p[2] = (i << (netbiasshift + 8)) / netsize; + freq[i] = intbias / netsize; /* 1/netsize */ + bias[i] = 0; + } + } + + public byte[] ColorMap() + { + byte[] map = new byte[3 * netsize]; + int[] index = new int[netsize]; + for (int i = 0; i < netsize; i++) + index[network[i][3]] = i; + int k = 0; + for (int i = 0; i < netsize; i++) { + int j = index[i]; + map[k++] = (byte)(network[j][0]); + map[k++] = (byte)(network[j][1]); + map[k++] = (byte)(network[j][2]); + } + return map; + } + + /// + /// Insertion sort of network and building of netindex[0..255] (to do after unbias) + /// + public void Inxbuild() + { + + int i, j, smallpos, smallval; + int[] p; + int[] q; + int previouscol, startpos; + + previouscol = 0; + startpos = 0; + for (i = 0; i < netsize; i++) { + p = network[i]; + smallpos = i; + smallval = p[1]; /* index on g */ + /* find smallest in i..netsize-1 */ + for (j = i + 1; j < netsize; j++) { + q = network[j]; + if (q[1] < smallval) { /* index on g */ + smallpos = j; + smallval = q[1]; /* index on g */ + } + } + q = network[smallpos]; + /* swap p (i) and q (smallpos) entries */ + if (i != smallpos) { + j = q[0]; + q[0] = p[0]; + p[0] = j; + j = q[1]; + q[1] = p[1]; + p[1] = j; + j = q[2]; + q[2] = p[2]; + p[2] = j; + j = q[3]; + q[3] = p[3]; + p[3] = j; + } + /* smallval entry is now in position i */ + if (smallval != previouscol) { + netindex[previouscol] = (startpos + i) >> 1; + for (j = previouscol + 1; j < smallval; j++) + netindex[j] = i; + previouscol = smallval; + startpos = i; + } + } + netindex[previouscol] = (startpos + maxnetpos) >> 1; + for (j = previouscol + 1; j < 256; j++) + netindex[j] = maxnetpos; /* really 256 */ + } + + /// + /// Main Learning Loop + /// + public void Learn() + { + + int i, j, b, g, r; + int radius, rad, alpha, step, delta, samplepixels; + byte[] p; + int pix, lim; + + if (lengthcount < minpicturebytes) + samplefac = 1; + alphadec = 30 + ((samplefac - 1) / 3); + p = thepicture; + pix = 0; + lim = lengthcount; + samplepixels = lengthcount / (3 * samplefac); + delta = samplepixels / ncycles; + alpha = initalpha; + radius = initradius; + + rad = radius >> radiusbiasshift; + if (rad <= 1) + rad = 0; + for (i = 0; i < rad; i++) + radpower[i] = + alpha * (((rad * rad - i * i) * radbias) / (rad * rad)); + + //fprintf(stderr,"beginning 1D learning: initial radius=%d\n", rad); + + if (lengthcount < minpicturebytes) + step = 3; + else if ((lengthcount % prime1) != 0) + step = 3 * prime1; + else { + if ((lengthcount % prime2) != 0) + step = 3 * prime2; + else { + if ((lengthcount % prime3) != 0) + step = 3 * prime3; + else + step = 3 * prime4; + } + } + + i = 0; + while (i < samplepixels) { + b = (p[pix + 0] & 0xff) << netbiasshift; + g = (p[pix + 1] & 0xff) << netbiasshift; + r = (p[pix + 2] & 0xff) << netbiasshift; + j = Contest(b, g, r); + + Altersingle(alpha, j, b, g, r); + if (rad != 0) + Alterneigh(rad, j, b, g, r); /* alter neighbours */ + + pix += step; + if (pix >= lim) + pix -= lengthcount; + + i++; + if (delta == 0) + delta = 1; + if (i % delta == 0) { + alpha -= alpha / alphadec; + radius -= radius / radiusdec; + rad = radius >> radiusbiasshift; + if (rad <= 1) + rad = 0; + for (j = 0; j < rad; j++) + radpower[j] = + alpha * (((rad * rad - j * j) * radbias) / (rad * rad)); + } + } + //fprintf(stderr,"finished 1D learning: readonly alpha=%f !\n",((float)alpha)/initalpha); + } + + /// + /// Search for BGR values 0..255 (after net is unbiased) and return colour index + /// + /// + /// + /// + /// + public int Map(int b, int g, int r) + { + + int i, j, dist, a, bestd; + int[] p; + int best; + + bestd = 1000; /* biggest possible dist is 256*3 */ + best = -1; + i = netindex[g]; /* index on g */ + j = i - 1; /* start at netindex[g] and work outwards */ + + while ((i < netsize) || (j >= 0)) { + if (i < netsize) { + p = network[i]; + dist = p[1] - g; /* inx key */ + if (dist >= bestd) + i = netsize; /* stop iter */ + else { + i++; + if (dist < 0) + dist = -dist; + a = p[0] - b; + if (a < 0) + a = -a; + dist += a; + if (dist < bestd) { + a = p[2] - r; + if (a < 0) + a = -a; + dist += a; + if (dist < bestd) { + bestd = dist; + best = p[3]; + } + } + } + } + if (j >= 0) { + p = network[j]; + dist = g - p[1]; /* inx key - reverse dif */ + if (dist >= bestd) + j = -1; /* stop iter */ + else { + j--; + if (dist < 0) + dist = -dist; + a = p[0] - b; + if (a < 0) + a = -a; + dist += a; + if (dist < bestd) { + a = p[2] - r; + if (a < 0) + a = -a; + dist += a; + if (dist < bestd) { + bestd = dist; + best = p[3]; + } + } + } + } + } + return (best); + } + public byte[] Process() + { + Learn(); + Unbiasnet(); + Inxbuild(); + return ColorMap(); + } + + /// + /// Unbias network to give byte values 0..255 and record position i to prepare for sort + /// + public void Unbiasnet() + { + + int i; + + for (i = 0; i < netsize; i++) { + network[i][0] >>= netbiasshift; + network[i][1] >>= netbiasshift; + network[i][2] >>= netbiasshift; + network[i][3] = i; /* record colour no */ + } + } + + /// + /// Move adjacent neurons by precomputed alpha*(1-((i-j)^2/[r]^2)) in radpower[|i-j|] + /// + /// + /// + /// + /// + /// + protected void Alterneigh(int rad, int i, int b, int g, int r) + { + + int j, k, lo, hi, a, m; + int[] p; + + lo = i - rad; + if (lo < -1) + lo = -1; + hi = i + rad; + if (hi > netsize) + hi = netsize; + + j = i + 1; + k = i - 1; + m = 1; + while ((j < hi) || (k > lo)) { + a = radpower[m++]; + if (j < hi) { + p = network[j++]; + try { + p[0] -= (a * (p[0] - b)) / alpharadbias; + p[1] -= (a * (p[1] - g)) / alpharadbias; + p[2] -= (a * (p[2] - r)) / alpharadbias; + } catch (Exception) { + } // prevents 1.3 miscompilation + } + if (k > lo) { + p = network[k--]; + try { + p[0] -= (a * (p[0] - b)) / alpharadbias; + p[1] -= (a * (p[1] - g)) / alpharadbias; + p[2] -= (a * (p[2] - r)) / alpharadbias; + } catch (Exception) { + } + } + } + } + + /// + /// Move neuron i towards biased (b,g,r) by factor alpha + /// + /// + /// + /// + /// + /// + protected void Altersingle(int alpha, int i, int b, int g, int r) + { + + /* alter hit neuron */ + int[] n = network[i]; + n[0] -= (alpha * (n[0] - b)) / initalpha; + n[1] -= (alpha * (n[1] - g)) / initalpha; + n[2] -= (alpha * (n[2] - r)) / initalpha; + } + + /// + /// Search for biased BGR values + /// + /// + /// + /// + /// + protected int Contest(int b, int g, int r) + { + + /* finds closest neuron (min dist) and updates freq */ + /* finds best neuron (min dist-bias) and returns position */ + /* for frequently chosen neurons, freq[i] is high and bias[i] is negative */ + /* bias[i] = gamma*((1/netsize)-freq[i]) */ + + int i, dist, a, biasdist, betafreq; + int bestpos, bestbiaspos, bestd, bestbiasd; + int[] n; + + bestd = ~(((int)1) << 31); + bestbiasd = bestd; + bestpos = -1; + bestbiaspos = bestpos; + + for (i = 0; i < netsize; i++) { + n = network[i]; + dist = n[0] - b; + if (dist < 0) + dist = -dist; + a = n[1] - g; + if (a < 0) + a = -a; + dist += a; + a = n[2] - r; + if (a < 0) + a = -a; + dist += a; + if (dist < bestd) { + bestd = dist; + bestpos = i; + } + biasdist = dist - ((bias[i]) >> (intbiasshift - netbiasshift)); + if (biasdist < bestbiasd) { + bestbiasd = biasdist; + bestbiaspos = i; + } + betafreq = (freq[i] >> betashift); + freq[i] -= betafreq; + bias[i] += (betafreq << gammashift); + } + freq[bestpos] += beta; + bias[bestpos] -= betagamma; + return (bestbiaspos); + } + } + } + } + } + } +} diff --git a/src/TorchSharp/Utils/tensorboard/Summary.cs b/src/TorchSharp/Utils/tensorboard/Summary.cs new file mode 100644 index 000000000..dd43d825d --- /dev/null +++ b/src/TorchSharp/Utils/tensorboard/Summary.cs @@ -0,0 +1,325 @@ +// Copyright (c) .NET Foundation and Contributors. All Rights Reserved. See LICENSE in the project root for license information. +using System; +using System.IO; +using Google.Protobuf; +using SkiaSharp; + +namespace TorchSharp +{ + public static partial class torch + { + public static partial class utils + { + public static partial class tensorboard + { + public static partial class Summary + { + private static int calc_scale_factor(Tensor tensor) + => tensor.dtype == ScalarType.Byte || tensor.dtype == ScalarType.Int8 ? 1 : 255; + + /// + /// Outputs a `Summary` protocol buffer with a histogram. + /// The generated + /// [`Summary`](https://www.tensorflow.org/code/tensorflow/core/framework/summary.proto) + /// has one summary value containing a histogram for `values`. + /// This op reports an `InvalidArgument` error if any value is not finite. + /// + /// https://github.com/pytorch/pytorch/blob/1.7/torch/utils/tensorboard/summary.py#L283 + /// + /// A name for the generated node. Will also serve as a series name in TensorBoard. + /// A real numeric `Tensor`. Any shape. Values to use to build the histogram. + /// + /// + /// A scalar `Tensor` of type `string`. The serialized `Summary` protocol buffer. + public static Tensorboard.Summary histogram(string name, Tensor values, Tensor bins, long? max_bins = null) + { + if (values.device.type != DeviceType.CPU) + values = values.cpu(); + Tensorboard.HistogramProto hist = make_histogram(values, bins, max_bins); + var summary = new Tensorboard.Summary(); + summary.Value.Add(new Tensorboard.Summary.Types.Value() { Tag = name, Histo = hist }); + return summary; + } + + /// + /// Outputs a `Summary` protocol buffer with a histogram. + /// The generated + /// [`Summary`](https://www.tensorflow.org/code/tensorflow/core/framework/summary.proto) + /// has one summary value containing a histogram for `values`. + /// This op reports an `InvalidArgument` error if any value is not finite. + /// + /// https://github.com/pytorch/pytorch/blob/1.7/torch/utils/tensorboard/summary.py#L283 + /// + /// A name for the generated node. Will also serve as a series name in TensorBoard. + /// A real numeric `Tensor`. Any shape. Values to use to build the histogram. + /// + /// + /// A scalar `Tensor` of type `string`. The serialized `Summary` protocol buffer. + public static Tensorboard.Summary histogram(string name, Tensor values, HistogramBinSelector bins, long? max_bins = null) + { + Tensorboard.HistogramProto hist = make_histogram(values, bins, max_bins); + var summary = new Tensorboard.Summary(); + summary.Value.Add(new Tensorboard.Summary.Types.Value() { Tag = name, Histo = hist }); + return summary; + } + + /// + /// Convert values into a histogram proto using logic from histogram.cc. + /// + /// https://github.com/pytorch/pytorch/blob/1.7/torch/utils/tensorboard/summary.py#L304 + /// + /// + /// + /// + /// + /// + public static Tensorboard.HistogramProto make_histogram(Tensor values, Tensor bins, long? max_bins = null) + { + if (values.numel() == 0) + throw new ArgumentException("The input has no element."); + if (values.dtype != ScalarType.Float64) + values = values.to_type(ScalarType.Float64); + values = values.reshape(-1); + (Tensor counts, Tensor limits) = torch.histogram(values, bins); + return make_histogram(values, counts, limits, max_bins); + } + + /// + /// Convert values into a histogram proto using logic from histogram.cc. + /// + /// https://github.com/pytorch/pytorch/blob/1.7/torch/utils/tensorboard/summary.py#L304 + /// + /// + /// + /// + /// + /// + public static Tensorboard.HistogramProto make_histogram(Tensor values, HistogramBinSelector bins, long? max_bins = null) + { + if (values.numel() == 0) + throw new ArgumentException("The input has no element."); + if (values.dtype != ScalarType.Float64) + values = values.to_type(ScalarType.Float64); + values = values.reshape(-1); + (Tensor counts, Tensor limits) = torch.histogram(values, bins); + return make_histogram(values, counts, limits, max_bins); + } + + private static Tensorboard.HistogramProto make_histogram(Tensor values, Tensor counts, Tensor limits, long? max_bins = null) + { + long num_bins = counts.shape[0]; + if (max_bins != null && num_bins > max_bins) { + long subsampling = num_bins / max_bins.Value; + long subsampling_remainder = num_bins % subsampling; + if (subsampling_remainder != 0) + counts = nn.functional.pad(counts, (0, subsampling - subsampling_remainder), PaddingModes.Constant, 0); + counts = counts.reshape(-1, subsampling).sum(1); + Tensor new_limits = empty(new long[] { counts.numel() + 1 }, limits.dtype); + new_limits[TensorIndex.Slice(null, -1)] = limits[TensorIndex.Slice(null, -1, subsampling)]; + new_limits[-1] = limits[-1]; + limits = new_limits; + } + + Tensor cum_counts = cumsum(greater(counts, 0), 0); + Tensor search_value = empty(2, cum_counts.dtype); + search_value[0] = 0; + search_value[1] = cum_counts[-1] - 1; + Tensor search_result = searchsorted(cum_counts, search_value, right: true); + long start = search_result[0].item(); + long end = search_result[1].item() + 1; + counts = start > 0 ? counts[TensorIndex.Slice(start - 1, end)] : concatenate(new Tensor[] { tensor(new[] { 0 }), counts[TensorIndex.Slice(stop: end)] }); + limits = limits[TensorIndex.Slice(start, end + 1)]; + + if (counts.numel() == 0 || limits.numel() == 0) + throw new ArgumentException("The histogram is empty, please file a bug report."); + + Tensor sum_sq = values.dot(values); + Tensorboard.HistogramProto histogramProto = new Tensorboard.HistogramProto() { + Min = values.min().item(), + Max = values.max().item(), + Num = values.shape[0], + Sum = values.sum().item(), + SumSquares = sum_sq.item(), + }; + histogramProto.BucketLimit.AddRange(limits.data().ToArray()); + histogramProto.Bucket.AddRange(counts.data().ToArray()); + return histogramProto; + } + + /// + /// Outputs a `Summary` protocol buffer with images. + /// The summary has up to `max_images` summary values containing images. The + /// images are built from `tensor` which must be 3-D with shape `[height, width, + /// channels]` and where `channels` can be: + /// * 1: `tensor` is interpreted as Grayscale. + /// * 3: `tensor` is interpreted as RGB. + /// * 4: `tensor` is interpreted as RGBA. + /// The `name` in the outputted Summary.Value protobufs is generated based on the + /// name, with a suffix depending on the max_outputs setting: + /// * If `max_outputs` is 1, the summary value tag is '*name*/image'. + /// * If `max_outputs` is greater than 1, the summary value tags are + /// generated sequentially as '*name*/image/0', '*name*/image/1', etc. + /// + /// A name for the generated node. Will also serve as a series name in TensorBoard. + /// + /// A 3-D `uint8` or `float32` `Tensor` of shape `[height, width, + /// channels]` where `channels` is 1, 3, or 4. + /// 'tensor' can either have values in [0, 1] (float32) or [0, 255] (uint8). + /// The image() function will scale the image values to [0, 255] by applying + /// a scale factor of either 1 (uint8) or 255 (float32). Out-of-range values + /// will be clipped. + /// + /// Rescale image size + /// Image data format specification of the form CHW, HWC, HW, WH, etc. + /// A scalar `Tensor` of type `string`. The serialized `Summary` protocol buffer. + public static Tensorboard.Summary image(string tag, Tensor tensor, double rescale = 1, string dataformats = "NCHW") + { + tensor = utils.convert_to_HWC(tensor, dataformats); + int scale_factor = calc_scale_factor(tensor); + tensor = tensor.to_type(ScalarType.Float32); + tensor = (tensor * scale_factor).clip(0, 255).to_type(ScalarType.Byte); + Tensorboard.Summary.Types.Image image = make_image(tensor, rescale); + var summary = new Tensorboard.Summary(); + summary.Value.Add(new Tensorboard.Summary.Types.Value() { Tag = tag, Image = image }); + return summary; + } + + /// + /// Outputs a `Summary` protocol buffer with images. + /// + /// A name for the generated node. Will also serve as a series name in TensorBoard. + /// local image filename + /// Rescale image size + /// A scalar `Tensor` of type `string`. The serialized `Summary` protocol buffer. + public static Tensorboard.Summary image(string tag, string file_name, double rescale = 1) + { + using var img = SKBitmap.Decode(file_name); + Tensorboard.Summary.Types.Image image = make_image(img, rescale); + var summary = new Tensorboard.Summary(); + summary.Value.Add(new Tensorboard.Summary.Types.Value() { Tag = tag, Image = image }); + return summary; + } + + /// + /// Convert a tensor representation of an image to Image protobuf + /// + /// https://github.com/pytorch/pytorch/blob/master/torch/utils/tensorboard/summary.py#L481 + /// + /// HWC(0~255) image tensor + /// Rescale image size + /// + public static Tensorboard.Summary.Types.Image make_image(Tensor tensor, double rescale = 1) + { + using SKBitmap skBmp = TensorToSKBitmap(tensor); + return make_image(skBmp, rescale); + } + + /// + /// Convert an image to Image protobuf + /// + /// https://github.com/pytorch/pytorch/blob/master/torch/utils/tensorboard/summary.py#L495 + /// + /// Image + /// Rescale image size + /// + internal static Tensorboard.Summary.Types.Image make_image(SKBitmap img, double rescale = 1) + { + using var image = img.Copy(); + byte[] bmpData = image.Resize(new SKSizeI((int)(image.Width * rescale), (int)(image.Height * rescale)), SKFilterQuality.High).Encode(SKEncodedImageFormat.Png, 100).ToArray(); + return new Tensorboard.Summary.Types.Image() { Height = image.Height, Width = image.Width, Colorspace = 4, EncodedImageString = ByteString.CopyFrom(bmpData) }; + } + + /// + /// https://github.com/pytorch/pytorch/blob/master/torch/utils/tensorboard/summary.py#L509 + /// + /// A name for the generated node. Will also serve as a series name in TensorBoard. + /// Video data + /// Frames per second + /// + public static Tensorboard.Summary video(string tag, Tensor tensor, int fps) + { + tensor = utils.prepare_video(tensor); + int scale_factor = calc_scale_factor(tensor); + tensor = tensor.to_type(ScalarType.Float32); + tensor = (tensor * scale_factor).clip(0, 255).to_type(ScalarType.Byte); + Tensorboard.Summary.Types.Image video = make_video(tensor, fps); + var summary = new Tensorboard.Summary(); + summary.Value.Add(new Tensorboard.Summary.Types.Value() { Tag = tag, Image = video }); + return summary; + } + + /// + /// https://github.com/pytorch/pytorch/blob/master/torch/utils/tensorboard/summary.py#L520 + /// + /// Video data + /// Frames per second + /// + public static Tensorboard.Summary.Types.Image make_video(Tensor tensor, int fps) + { + int h = (int)tensor.shape[1]; + int w = (int)tensor.shape[2]; + using GifEncoder.Encoder encoder = new GifEncoder.Encoder(); + encoder.Start(); + encoder.SetRepeat(0); + encoder.SetFrameRate(fps); + foreach (var t in tensor.split(1)) { + using SKBitmap bitmap = TensorToSKBitmap(t.squeeze()); + encoder.AddFrame(bitmap); + } + encoder.Finish(); + Stream stream = encoder.Output(); + stream.Position = 0; + return new Tensorboard.Summary.Types.Image() { Height = h, Width = w, Colorspace = 4, EncodedImageString = ByteString.FromStream(stream) }; + } + + private static SKBitmap TensorToSKBitmap(Tensor tensor) + { + int h = (int)tensor.shape[0]; + int w = (int)tensor.shape[1]; + int c = (int)tensor.shape[2]; + + byte[,,] data = tensor.cpu().data().ToNDArray() as byte[,,]; + var skBmp = new SKBitmap(w, h, SKColorType.Rgba8888, SKAlphaType.Opaque); + int pixelSize = 4; + unsafe { + byte* pSkBmp = (byte*)skBmp.GetPixels().ToPointer(); + for (int i = 0; i < h; i++) { + for (int j = 0; j < w; j++) { + pSkBmp[j * pixelSize] = data[i, j, 0]; + pSkBmp[j * pixelSize + 1] = data[i, j, 1]; + pSkBmp[j * pixelSize + 2] = data[i, j, 2]; + pSkBmp[j * pixelSize + 3] = c == 4 ? data[i, j, 3] : (byte)255; + } + pSkBmp += skBmp.Info.RowBytes; + } + } + + return skBmp; + } + + /// + /// https://github.com/pytorch/pytorch/blob/master/torch/utils/tensorboard/summary.py#L630 + /// + /// Data identifier + /// String to save + /// + public static Tensorboard.Summary text(string tag, string text) + { + // TextPluginData(version=0).SerializeToString() + // output: b'' + var pluginData = new Tensorboard.SummaryMetadata.Types.PluginData() { PluginName = "text", Content = ByteString.CopyFromUtf8("") }; + var smd = new Tensorboard.SummaryMetadata() { PluginData = pluginData }; + var shapeProto = new Tensorboard.TensorShapeProto(); + shapeProto.Dim.Add(new Tensorboard.TensorShapeProto.Types.Dim() { Size = 1 }); + var tensor = new Tensorboard.TensorProto() { Dtype = Tensorboard.DataType.DtString, TensorShape = shapeProto }; + tensor.StringVal.Add(ByteString.CopyFromUtf8(text)); + + var summary = new Tensorboard.Summary(); + summary.Value.Add(new Tensorboard.Summary.Types.Value() { Tag = tag + "/text_summary", Metadata = smd, Tensor = tensor }); + return summary; + } + } + } + } + } +} diff --git a/src/TorchSharp/Utils/tensorboard/SummaryWriter.cs b/src/TorchSharp/Utils/tensorboard/SummaryWriter.cs index a892502e3..3dc444bec 100644 --- a/src/TorchSharp/Utils/tensorboard/SummaryWriter.cs +++ b/src/TorchSharp/Utils/tensorboard/SummaryWriter.cs @@ -1,9 +1,9 @@ // Copyright (c) .NET Foundation and Contributors. All Rights Reserved. See LICENSE in the project root for license information. using System; +using System.Collections.Generic; using System.Diagnostics; using System.IO; -using System.Collections.Generic; using System.Threading; using Google.Protobuf; using Tensorboard; @@ -104,17 +104,12 @@ internal SummaryWriter(string log_dir, string filename_suffix) /// Optional override default walltime (DateTimeOffset.Now.ToUnixTimeSeconds()) public void add_scalar(string tag, float scalar_value, int global_step, long? walltime = null) { - var fileName = _fileNames["__default__"]; - - if (!File.Exists(fileName)) { - InitFile(fileName); - } - - var wt = walltime.HasValue ? walltime.Value : DateTimeOffset.Now.ToUnixTimeSeconds(); + var fileName = InitDefaultFile(); + SetWalltime(ref walltime); var summary = new Summary(); summary.Value.Add(new Summary.Types.Value() { SimpleValue = scalar_value, Tag = tag }); - var evnt = new Event() { Step = global_step, WallTime = wt, Summary = summary }; + var evnt = new Event() { Step = global_step, WallTime = walltime.Value, Summary = summary }; WriteEvent(fileName, evnt); } @@ -128,7 +123,7 @@ public void add_scalar(string tag, float scalar_value, int global_step, long? wa /// Optional override default walltime (DateTimeOffset.Now.ToUnixTimeSeconds()) public void add_scalars(string main_tag, IDictionary tag_scalar_dict, int global_step, long? walltime = null) { - var wt = walltime.HasValue ? walltime.Value : DateTimeOffset.Now.ToUnixTimeSeconds(); + SetWalltime(ref walltime); foreach (var kv in tag_scalar_dict) { @@ -137,7 +132,7 @@ public void add_scalars(string main_tag, IDictionary tag_scalar_d var summary = new Summary(); summary.Value.Add(new Summary.Types.Value() { SimpleValue = scalar_value, Tag = main_tag }); - var evnt = new Event() { Step = global_step, WallTime = wt, Summary = summary }; + var evnt = new Event() { Step = global_step, WallTime = walltime.Value, Summary = summary }; if (!_fileNames.TryGetValue(key, out var fileName)) { @@ -173,13 +168,13 @@ public void add_scalars(string main_tag, IDictionary tag_scalar_d /// Optional override default walltime (DateTimeOffset.Now.ToUnixTimeSeconds()) public void add_scalars(string main_tag, IList<(string, float)> tag_scalar_dict, int global_step, long? walltime = null) { - var wt = walltime.HasValue ? walltime.Value : DateTimeOffset.Now.ToUnixTimeSeconds(); + SetWalltime(ref walltime); foreach (var (key, scalar_value) in tag_scalar_dict) { var summary = new Summary(); summary.Value.Add(new Summary.Types.Value() { SimpleValue = scalar_value, Tag = main_tag }); - var evnt = new Event() { Step = global_step, WallTime = wt, Summary = summary }; + var evnt = new Event() { Step = global_step, WallTime = walltime.Value, Summary = summary }; if (!_fileNames.TryGetValue(key, out var fileName)) { @@ -201,12 +196,147 @@ public void add_scalars(string main_tag, IList<(string, float)> tag_scalar_dict, } } + /// + /// Add histogram to summary. + /// + /// https://pytorch.org/docs/stable/_modules/torch/utils/tensorboard/writer.html#SummaryWriter.add_histogram + /// + /// Data identifier + /// Values to build histogram + /// Global step value to record + /// This determines how the bins are made + /// Optional override default walltime (DateTimeOffset.Now.ToUnixTimeSeconds()) + /// + public void add_histogram(string tag, + torch.Tensor values, + int global_step, + Utils.tensorboard.Enums.HistogramBinSelector bins = Utils.tensorboard.Enums.HistogramBinSelector.Tensorflow, + long? walltime = null, + long? max_bins = null) + { + static torch.Tensor default_bins() + { + double v = 1e-12; + var buckets = new List(); + var neg_buckets = new List(); + while (v < 1e20) { + buckets.Add(v); + neg_buckets.Add(-v); + v *= 1.1; + } + neg_buckets.Reverse(); + var result = new List(); + result.AddRange(neg_buckets); result.Add(0); result.AddRange(buckets); + return torch.tensor(result); + } + + var fileName = InitDefaultFile(); + SetWalltime(ref walltime); + Summary summary = bins == Utils.tensorboard.Enums.HistogramBinSelector.Tensorflow ? + torch.utils.tensorboard.Summary.histogram(tag, values, default_bins(), max_bins) : + torch.utils.tensorboard.Summary.histogram(tag, values, (HistogramBinSelector)(byte)bins, max_bins); + var evnt = new Event() { Step = global_step, WallTime = walltime.Value, Summary = summary }; + WriteEvent(fileName, evnt); + } + + /// + /// Add batched image data to summary. + /// + /// https://pytorch.org/docs/stable/tensorboard.html#torch.utils.tensorboard.writer.SummaryWriter.add_image + /// + /// Data identifier + /// Image data + /// Global step value to record + /// Optional override default walltime (DateTimeOffset.Now.ToUnixTimeSeconds()) + /// Image data format specification of the form CHW, HWC, HW, WH, etc. + public void add_img(string tag, torch.Tensor img_tensor, int global_step, long? walltime = null, string dataformats = "CHW") + { + var fileName = InitDefaultFile(); + SetWalltime(ref walltime); + Summary summary = torch.utils.tensorboard.Summary.image(tag, img_tensor, dataformats: dataformats); + var evnt = new Event() { Step = global_step, WallTime = walltime.Value, Summary = summary }; + WriteEvent(fileName, evnt); + } + + /// + /// Add batched image data to summary. + /// + /// https://pytorch.org/docs/stable/tensorboard.html#torch.utils.tensorboard.writer.SummaryWriter.add_image + /// + /// Data identifier + /// Image file + /// Global step value to record + /// Optional override default walltime (DateTimeOffset.Now.ToUnixTimeSeconds()) + public void add_img(string tag, string file_name, int global_step, long? walltime = null) + { + var fileName = InitDefaultFile(); + SetWalltime(ref walltime); + Summary summary = torch.utils.tensorboard.Summary.image(tag, file_name); + var evnt = new Event() { Step = global_step, WallTime = walltime.Value, Summary = summary }; + WriteEvent(fileName, evnt); + } + + /// + /// Add video data to summary. + /// + /// https://pytorch.org/docs/stable/tensorboard.html#torch.utils.tensorboard.writer.SummaryWriter.add_video + /// + /// Data identifier + /// + /// Video data + /// + /// Shape: (N,T,C,H,W). The values should lie in [0, 255] for type uint8 or [0, 1] for type float. + /// + /// Global step value to record + /// Frames per second + /// Optional override default walltime (DateTimeOffset.Now.ToUnixTimeSeconds()) + public void add_video(string tag, torch.Tensor vid_tensor, int global_step, int fps = 4, long? walltime = null) + { + var fileName = InitDefaultFile(); + SetWalltime(ref walltime); + Summary summary = torch.utils.tensorboard.Summary.video(tag, vid_tensor, fps); + var evnt = new Event() { Step = global_step, WallTime = walltime.Value, Summary = summary }; + WriteEvent(fileName, evnt); + } + + /// + /// Add text data to summary. + /// + /// https://pytorch.org/docs/stable/_modules/torch/utils/tensorboard/writer.html#SummaryWriter.add_text + /// + /// Data identifier + /// String to save + /// Global step value to record + /// Optional override default walltime (DateTimeOffset.Now.ToUnixTimeSeconds()) + public void add_text(string tag, string text_string, int global_step, long? walltime = null) + { + var fileName = InitDefaultFile(); + SetWalltime(ref walltime); + + var evnt = new Event() { Step = global_step, WallTime = walltime.Value, Summary = torch.utils.tensorboard.Summary.text(tag, text_string) }; + WriteEvent(fileName, evnt); + } + private static void InitFile(string fileName) { var evnt = new Event() { FileVersion = "brain.Event:2", WallTime = DateTime.Now.Ticks }; WriteEvent(fileName, evnt); } + private string InitDefaultFile() + { + var fileName = _fileNames["__default__"]; + + if (!File.Exists(fileName)) { + InitFile(fileName); + } + + return fileName; + } + + private static void SetWalltime(ref long? walltime) + => walltime ??= DateTimeOffset.Now.ToUnixTimeSeconds(); + private static void WriteEvent(string fileName, Event evnt) { var bytes = evnt.ToByteArray(); @@ -215,7 +345,7 @@ private static void WriteEvent(string fileName, Event evnt) uint header_crc = GetMaskedCrc(header); uint footer_crc = GetMaskedCrc(bytes); - using (var fStream = File.Open(fileName,FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read)) { + using (var fStream = File.Open(fileName, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read)) { fStream.Seek(0, SeekOrigin.End); using (var writers = new BinaryWriter(fStream)) { diff --git a/src/TorchSharp/Utils/tensorboard/Utils.cs b/src/TorchSharp/Utils/tensorboard/Utils.cs new file mode 100644 index 000000000..edfccd899 --- /dev/null +++ b/src/TorchSharp/Utils/tensorboard/Utils.cs @@ -0,0 +1,125 @@ +// Copyright (c) .NET Foundation and Contributors. All Rights Reserved. See LICENSE in the project root for license information. + +using System; +using System.Diagnostics; +using System.Linq; + +namespace TorchSharp +{ + public static partial class torch + { + public static partial class utils + { + public static partial class tensorboard + { + internal static partial class utils + { + /// + /// Converts a 5D tensor [batchsize, time(frame), channel(color), height, width] + /// into 4D tensor with dimension[time(frame), new_width, new_height, channel]. + /// A batch of images are spreaded to a grid, which forms a frame. + /// e.g. Video with batchsize 16 will have a 4x4 grid. + /// + /// https://github.com/pytorch/pytorch/blob/master/torch/utils/tensorboard/_utils.py#L110 + /// + /// + /// + public static Tensor prepare_video(Tensor V) + { + long b = V.shape[0]; + long t = V.shape[1]; + long c = V.shape[2]; + long h = V.shape[3]; + long w = V.shape[4]; + + if (V.dtype == ScalarType.Int8 || V.dtype == ScalarType.Byte) + V = V.to_type(ScalarType.Float32) / 255.0; + + bool is_power2(long num) + => num != 0 && ((num & (num - 1)) == 0); + int bit_length(long value) + => Convert.ToString(value, 2).Length; + + if (!is_power2(V.shape[0])) { + int len_addition = Convert.ToInt32(Math.Pow(2, bit_length(V.shape[0])) - V.shape[0]); + V = cat(new Tensor[] { V, zeros(new long[] { len_addition, t, c, h, w }, device: V.device) }); + } + + long n_rows = Convert.ToInt32(Math.Pow(2, (bit_length(V.shape[0]) - 1) / 2)); + long n_cols = V.shape[0] / n_rows; + + V = V.reshape(n_rows, n_cols, t, c, h, w); + V = V.permute(2, 0, 4, 1, 5, 3); + V = V.reshape(t, n_rows * h, n_cols * w, c); + return V; + } + + /// + /// https://github.com/pytorch/pytorch/blob/6c30dc6ceed5542351b3be4f8043b28020f93f3a/torch/utils/tensorboard/_utils.py#L69 + /// + /// + /// + /// + public static Tensor make_grid(Tensor I, long ncols = 8) + { + if (I.shape[1] == 1) + I = I.expand(-1, 3); + Trace.Assert(I.ndim == 4 && I.shape[1] == 3); + long nimg = I.shape[0]; + long H = I.shape[2]; + long W = I.shape[3]; + ncols = Math.Min(ncols, nimg); + long nrows = (long)Math.Ceiling((double)nimg / (double)ncols); + Tensor canvas = zeros(new long[] { 3, H * nrows, W * ncols }, I.dtype); + long i = 0; + for (long r = 0; r < nrows; r++) { + for (long c = 0; c < ncols; c++) { + if (i >= nimg) + break; + canvas.narrow(1, r * H, H).narrow(2, c * W, W).copy_(I[i]); + i++; + } + } + return canvas; + } + + /// + /// https://github.com/pytorch/pytorch/blob/6c30dc6ceed5542351b3be4f8043b28020f93f3a/torch/utils/tensorboard/_utils.py#L95 + /// + /// Image data + /// Image data format specification of the form NCHW, NHWC, CHW, HWC, HW, WH, etc. + /// + public static Tensor convert_to_HWC(Tensor tensor, string input_format) + { + Trace.Assert(tensor.shape.Length == input_format.Length, $"size of input tensor and input format are different. tensor shape: ({string.Join(", ", tensor.shape)}), input_format: {input_format}"); + input_format = input_format.ToUpper(); + + if (input_format.Length == 4) { + long[] index = "NCHW".Select(c => Convert.ToInt64(input_format.IndexOf(c))).ToArray(); + Tensor tensor_NCHW = tensor.permute(index); + Tensor tensor_CHW = make_grid(tensor_NCHW); + return tensor_CHW.permute(1, 2, 0); + } + + if (input_format.Length == 3) { + long[] index = "HWC".Select(c => Convert.ToInt64(input_format.IndexOf(c))).ToArray(); + Tensor tensor_HWC = tensor.permute(index); + if (tensor_HWC.shape[2] == 1) + tensor_HWC = tensor_HWC.expand(-1, -1, 3); + return tensor_HWC; + } + + if (input_format.Length == 2) { + long[] index = "HW".Select(c => Convert.ToInt64(input_format.IndexOf(c))).ToArray(); + tensor = tensor.permute(index); + tensor = tensor.unsqueeze(-1).expand(-1, -1, 3); + return tensor; + } + + throw new NotImplementedException(); + } + } + } + } + } +}