Skip to content

Commit b363322

Browse files
MpdreamzMpdreamz
committed
Add support for source only snapshot repository (#3531)
1 parent 801522e commit b363322

File tree

10 files changed

+225
-8
lines changed

10 files changed

+225
-8
lines changed

‎src/Nest/Ingest/Processors/Plugins/GeoIpProcessor.cs‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,6 @@ public class GeoIpProcessorDescriptor<T>
8282
IEnumerable<string> IGeoIpProcessor.Properties { get; set; }
8383
Field IGeoIpProcessor.TargetField { get; set; }
8484

85-
8685
public GeoIpProcessorDescriptor<T> Field(Field field) => Assign(a => a.Field = field);
8786

8887
public GeoIpProcessorDescriptor<T> Field(Expression<Func<T, object>> objectPath) =>

‎src/Nest/Modules/SnapshotAndRestore/Repositories/AzureRepository.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ public AzureRepository() { }
1212
public AzureRepository(IAzureRepositorySettings settings) => Settings = settings;
1313

1414
public IAzureRepositorySettings Settings { get; set; }
15+
object IRepositoryWithSettings.DelegateSettings => Settings;
1516
public string Type { get; } = "azure";
1617
}
1718

@@ -87,6 +88,7 @@ public class AzureRepositoryDescriptor
8788
{
8889
IAzureRepositorySettings IRepository<IAzureRepositorySettings>.Settings { get; set; }
8990
string ISnapshotRepository.Type { get; } = "azure";
91+
object IRepositoryWithSettings.DelegateSettings => Self.Settings;
9092

9193
public AzureRepositoryDescriptor Settings(Func<AzureRepositorySettingsDescriptor, IAzureRepositorySettings> settingsSelector) =>
9294
Assign(a => a.Settings = settingsSelector?.Invoke(new AzureRepositorySettingsDescriptor()));

‎src/Nest/Modules/SnapshotAndRestore/Repositories/CreateRepository/CreateRepositoryRequest.cs‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -40,18 +40,18 @@ public CreateRepositoryDescriptor ReadOnlyUrl(Func<ReadOnlyUrlRepositoryDescript
4040
public CreateRepositoryDescriptor Azure(Func<AzureRepositoryDescriptor, IAzureRepository> selector = null) =>
4141
Assign(a => a.Repository = selector.InvokeOrDefault(new AzureRepositoryDescriptor()));
4242

43-
/// <summary>
44-
/// Create an snapshot/restore repository that points to an HDFS filesystem
45-
/// </summary>
43+
/// <summary> Create an snapshot/restore repository that points to an HDFS filesystem </summary>
4644
public CreateRepositoryDescriptor Hdfs(Func<HdfsRepositoryDescriptor, IHdfsRepository> selector) =>
4745
Assign(a => a.Repository = selector?.Invoke(new HdfsRepositoryDescriptor()));
4846

49-
/// <summary>
50-
/// Snapshot and restore to an Amazon S3 bucket
51-
/// </summary>
47+
/// <summary> Snapshot and restore to an Amazon S3 bucket </summary>
5248
public CreateRepositoryDescriptor S3(Func<S3RepositoryDescriptor, IS3Repository> selector) =>
5349
Assign(a => a.Repository = selector?.Invoke(new S3RepositoryDescriptor()));
5450

51+
/// <summary> Snapshot and restore to an Amazon S3 bucket </summary>
52+
public CreateRepositoryDescriptor SourceOnly(Func<SourceOnlyRepositoryDescriptor, ISourceOnlyRepository> selector) =>
53+
Assign(a => a.Repository = selector?.Invoke(new SourceOnlyRepositoryDescriptor()));
54+
5555
/// <summary>
5656
/// Register a custom repository
5757
/// </summary>

‎src/Nest/Modules/SnapshotAndRestore/Repositories/FileSystemRepository.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ public class FileSystemRepository : IFileSystemRepository
1010
public FileSystemRepository(FileSystemRepositorySettings settings) => Settings = settings;
1111

1212
public IFileSystemRepositorySettings Settings { get; set; }
13+
object IRepositoryWithSettings.DelegateSettings => Settings;
1314
public string Type { get; } = "fs";
1415
}
1516

@@ -109,6 +110,7 @@ public class FileSystemRepositoryDescriptor
109110
: DescriptorBase<FileSystemRepositoryDescriptor, IFileSystemRepository>, IFileSystemRepository
110111
{
111112
IFileSystemRepositorySettings IRepository<IFileSystemRepositorySettings>.Settings { get; set; }
113+
object IRepositoryWithSettings.DelegateSettings => Self.Settings;
112114
string ISnapshotRepository.Type { get; } = "fs";
113115

114116
public FileSystemRepositoryDescriptor Settings(string location,

‎src/Nest/Modules/SnapshotAndRestore/Repositories/HdfsRepository.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ public class HdfsRepository : IHdfsRepository
1111
public HdfsRepository(HdfsRepositorySettings settings) => Settings = settings;
1212

1313
public IHdfsRepositorySettings Settings { get; set; }
14+
object IRepositoryWithSettings.DelegateSettings => Settings;
1415
public string Type { get; } = "hdfs";
1516
}
1617

@@ -127,6 +128,7 @@ public class HdfsRepositoryDescriptor
127128
: DescriptorBase<HdfsRepositoryDescriptor, IHdfsRepository>, IHdfsRepository
128129
{
129130
IHdfsRepositorySettings IRepository<IHdfsRepositorySettings>.Settings { get; set; }
131+
object IRepositoryWithSettings.DelegateSettings => Self.Settings;
130132
string ISnapshotRepository.Type => "hdfs";
131133

132134
public HdfsRepositoryDescriptor Settings(string path, Func<HdfsRepositorySettingsDescriptor, IHdfsRepositorySettings> settingsSelector = null

‎src/Nest/Modules/SnapshotAndRestore/Repositories/ISnapshotRepository.cs‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,13 @@ public interface ISnapshotRepository
88
string Type { get; }
99
}
1010

11-
public interface IRepository<TSettings> : ISnapshotRepository
11+
public interface IRepositoryWithSettings: ISnapshotRepository
12+
{
13+
[JsonIgnore]
14+
object DelegateSettings { get; }
15+
}
16+
17+
public interface IRepository<TSettings> : IRepositoryWithSettings
1218
where TSettings : class, IRepositorySettings
1319
{
1420
[JsonProperty("settings")]

‎src/Nest/Modules/SnapshotAndRestore/Repositories/ReadOnlyUrlRepository.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ public class ReadOnlyUrlRepository : IReadOnlyUrlRepository
1010
public ReadOnlyUrlRepository(ReadOnlyUrlRepositorySettings settings) => Settings = settings;
1111

1212
public IReadOnlyUrlRepositorySettings Settings { get; set; }
13+
object IRepositoryWithSettings.DelegateSettings => Settings;
1314
public string Type { get; } = "url";
1415
}
1516

@@ -57,6 +58,7 @@ public class ReadOnlyUrlRepositoryDescriptor
5758
: DescriptorBase<ReadOnlyUrlRepositoryDescriptor, IReadOnlyUrlRepository>, IReadOnlyUrlRepository
5859
{
5960
IReadOnlyUrlRepositorySettings IRepository<IReadOnlyUrlRepositorySettings>.Settings { get; set; }
61+
object IRepositoryWithSettings.DelegateSettings => Self.Settings;
6062
string ISnapshotRepository.Type => "url";
6163

6264
public ReadOnlyUrlRepositoryDescriptor Settings(string location,

‎src/Nest/Modules/SnapshotAndRestore/Repositories/S3Repository.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ public class S3Repository : IS3Repository
1010
public S3Repository(S3RepositorySettings settings) => Settings = settings;
1111

1212
public IS3RepositorySettings Settings { get; set; }
13+
object IRepositoryWithSettings.DelegateSettings => Settings;
1314
public string Type { get; } = "s3";
1415
}
1516

@@ -205,6 +206,7 @@ public class S3RepositoryDescriptor
205206
: DescriptorBase<S3RepositoryDescriptor, IS3Repository>, IS3Repository
206207
{
207208
IS3RepositorySettings IRepository<IS3RepositorySettings>.Settings { get; set; }
209+
object IRepositoryWithSettings.DelegateSettings => Self.Settings;
208210
string ISnapshotRepository.Type { get; } = "s3";
209211

210212
public S3RepositoryDescriptor Settings(string bucket, Func<S3RepositorySettingsDescriptor, IS3RepositorySettings> settingsSelector = null) =>

‎src/Nest/Modules/SnapshotAndRestore/Repositories/SourceOnlyRepository .cs‎

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
using System;
2+
using Newtonsoft.Json;
3+
using Newtonsoft.Json.Linq;
4+
5+
namespace Nest
6+
{
7+
8+
[JsonConverter(typeof(SourceOnlyRepositorySerializer))]
9+
public interface ISourceOnlyRepository : IRepositoryWithSettings
10+
{
11+
[JsonIgnore]
12+
string DelegateType { get; }
13+
}
14+
15+
public class SourceOnlyRepository : ISourceOnlyRepository
16+
{
17+
private readonly object _delegateSettings;
18+
private readonly string _delegateType;
19+
20+
internal SourceOnlyRepository() { }
21+
22+
internal SourceOnlyRepository(string delegateType, object settings)
23+
{
24+
_delegateType = delegateType;
25+
_delegateSettings = settings;
26+
}
27+
28+
public SourceOnlyRepository(IRepositoryWithSettings repositoryToDelegateTo)
29+
{
30+
if (repositoryToDelegateTo == null) throw new ArgumentNullException(nameof(repositoryToDelegateTo));
31+
32+
_delegateType = repositoryToDelegateTo.Type;
33+
_delegateSettings = repositoryToDelegateTo.DelegateSettings;
34+
}
35+
36+
object IRepositoryWithSettings.DelegateSettings => _delegateSettings;
37+
string ISourceOnlyRepository.DelegateType => _delegateType;
38+
string ISnapshotRepository.Type { get; } = "source";
39+
}
40+
41+
public class SourceOnlyRepositoryDescriptor
42+
: DescriptorBase<SourceOnlyRepositoryDescriptor, ISourceOnlyRepository>, ISourceOnlyRepository
43+
{
44+
private object _delegateSettings;
45+
private string _delegateType;
46+
47+
object IRepositoryWithSettings.DelegateSettings => _delegateSettings;
48+
string ISourceOnlyRepository.DelegateType => _delegateType;
49+
string ISnapshotRepository.Type { get; } = "source";
50+
51+
private SourceOnlyRepositoryDescriptor DelegateTo<TDescriptor>(Func<TDescriptor, IRepositoryWithSettings> selector)
52+
where TDescriptor : IRepositoryWithSettings, new() => Custom(selector?.Invoke(new TDescriptor()));
53+
54+
/// <inheritdoc cref="CreateRepositoryDescriptor.FileSystem" />
55+
public SourceOnlyRepositoryDescriptor FileSystem(Func<FileSystemRepositoryDescriptor, IFileSystemRepository> selector) =>
56+
DelegateTo(selector);
57+
58+
/// <inheritdoc cref="CreateRepositoryDescriptor.ReadOnlyUrl" />
59+
public SourceOnlyRepositoryDescriptor ReadOnlyUrl(Func<ReadOnlyUrlRepositoryDescriptor, IReadOnlyUrlRepository> selector) =>
60+
DelegateTo(selector);
61+
62+
/// <inheritdoc cref="CreateRepositoryDescriptor.ReadOnlyUrl" />
63+
public SourceOnlyRepositoryDescriptor Azure(Func<AzureRepositoryDescriptor, IAzureRepository> selector = null) =>
64+
DelegateTo(selector);
65+
66+
/// <inheritdoc cref="CreateRepositoryDescriptor.ReadOnlyUrl" />
67+
public SourceOnlyRepositoryDescriptor Hdfs(Func<HdfsRepositoryDescriptor, IHdfsRepository> selector) =>
68+
DelegateTo(selector);
69+
70+
/// <inheritdoc cref="CreateRepositoryDescriptor.ReadOnlyUrl" />
71+
public SourceOnlyRepositoryDescriptor S3(Func<S3RepositoryDescriptor, IS3Repository> selector) =>
72+
DelegateTo(selector);
73+
74+
/// <inheritdoc cref="CreateRepositoryDescriptor.ReadOnlyUrl" />
75+
public SourceOnlyRepositoryDescriptor Custom(IRepositoryWithSettings repository)
76+
{
77+
_delegateType = repository?.Type;
78+
_delegateSettings = repository?.DelegateSettings;
79+
return this;
80+
}
81+
}
82+
83+
internal class SourceOnlyRepositorySerializer : ReserializeJsonConverter<SourceOnlyRepository, ISourceOnlyRepository>
84+
{
85+
protected override void SerializeJson(JsonWriter writer, object value, ISourceOnlyRepository castValue, JsonSerializer serializer)
86+
{
87+
if (castValue.DelegateType.IsNullOrEmpty())
88+
{
89+
writer.WriteNull();
90+
return;
91+
}
92+
writer.WriteStartObject();
93+
writer.WriteProperty(serializer, "type", "source");
94+
if (castValue.DelegateSettings != null)
95+
{
96+
writer.WritePropertyName("settings");
97+
writer.WriteStartObject();
98+
writer.WriteProperty(serializer, "delegate_type", castValue.DelegateType);
99+
var properties = castValue.DelegateSettings.GetType().GetCachedObjectProperties();
100+
foreach (var p in properties)
101+
{
102+
if (p.Ignored) continue;
103+
104+
var vv = p.ValueProvider.GetValue(castValue.DelegateSettings);
105+
if (vv == null) continue;
106+
107+
writer.WritePropertyName(p.PropertyName);
108+
serializer.Serialize(writer, vv);
109+
}
110+
writer.WriteEndObject();
111+
}
112+
writer.WriteEndObject();
113+
}
114+
115+
public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
116+
{
117+
if (reader.TokenType != JsonToken.StartObject) return null;
118+
119+
var o = JObject.Load(reader);
120+
if (o == null) return null;
121+
122+
if (!o.TryGetValue("settings", out var token))
123+
return null;
124+
125+
if (!(token is JObject settingsObject))
126+
return null;
127+
128+
if (!settingsObject.TryGetValue("delegate_type", out var delegateTypeToken))
129+
return null;
130+
131+
settingsObject.Remove("delegate_type");
132+
133+
var settings = settingsObject.ToObject<object>(serializer);
134+
return new SourceOnlyRepository(delegateTypeToken.Value<string>(), settings);
135+
}
136+
}
137+
}

‎src/Tests/Tests/Modules/SnapshotAndRestore/Repositories/CreateRepository/CreateRepositoryApiTests.cs‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using System;
2+
using Elastic.Xunit.XunitPlumbing;
23
using Elasticsearch.Net;
34
using Nest;
45
using Tests.Core.ManagedElasticsearch.Clusters;
@@ -128,6 +129,70 @@ protected override LazyResponses ClientUsage() => Calls(
128129
protected override CreateRepositoryDescriptor NewDescriptor() => new CreateRepositoryDescriptor(_name);
129130
}
130131

132+
[SkipVersion("<6.5.0", "new feature")]
133+
public class CreateSourceOnlyRepositoryApiTests
134+
: ApiTestBase<WritableCluster, ICreateRepositoryResponse, ICreateRepositoryRequest, CreateRepositoryDescriptor, CreateRepositoryRequest>
135+
{
136+
private static readonly string _name = "repository10";
137+
138+
public CreateSourceOnlyRepositoryApiTests(WritableCluster cluster, EndpointUsage usage) : base(cluster, usage) { }
139+
140+
protected override object ExpectJson { get; } = new
141+
{
142+
type = "source",
143+
settings = new
144+
{
145+
delegate_type = "fs",
146+
location = "some/location",
147+
compress = true,
148+
concurrent_streams = 5,
149+
chunk_size = "64mb",
150+
max_restore_bytes_per_second = "100mb",
151+
max_snapshot_bytes_per_second = "200mb"
152+
}
153+
};
154+
155+
protected override Func<CreateRepositoryDescriptor, ICreateRepositoryRequest> Fluent => d => d
156+
.SourceOnly(o => o
157+
.FileSystem(fs=>fs
158+
.Settings("some/location", s => s
159+
.Compress()
160+
.ConcurrentStreams(5)
161+
.ChunkSize("64mb")
162+
.RestoreBytesPerSecondMaximum("100mb")
163+
.SnapshotBytesPerSecondMaximum("200mb")
164+
)
165+
)
166+
);
167+
168+
protected override HttpMethod HttpMethod => HttpMethod.PUT;
169+
170+
protected override CreateRepositoryRequest Initializer => new CreateRepositoryRequest(_name)
171+
{
172+
Repository = new SourceOnlyRepository(new FileSystemRepository(new FileSystemRepositorySettings("some/location")
173+
{
174+
Compress = true,
175+
ConcurrentStreams = 5,
176+
ChunkSize = "64mb",
177+
RestoreBytesPerSecondMaximum = "100mb",
178+
SnapshotBytesPerSecondMaximum = "200mb"
179+
}))
180+
};
181+
182+
protected override bool SupportsDeserialization => false;
183+
protected override string UrlPath => $"/_snapshot/{_name}";
184+
185+
protected override LazyResponses ClientUsage() => Calls(
186+
(client, f) => client.CreateRepository(_name, f),
187+
(client, f) => client.CreateRepositoryAsync(_name, f),
188+
(client, r) => client.CreateRepository(r),
189+
(client, r) => client.CreateRepositoryAsync(r)
190+
);
191+
192+
protected override CreateRepositoryDescriptor NewDescriptor() => new CreateRepositoryDescriptor(_name);
193+
}
194+
195+
131196
public class CreateFileSystemRepositoryApiTests
132197
: ApiTestBase<WritableCluster, ICreateRepositoryResponse, ICreateRepositoryRequest, CreateRepositoryDescriptor, CreateRepositoryRequest>
133198
{

0 commit comments

Comments
 (0)