From b511d832c7b546e9c41e7aa7a3068868ba797666 Mon Sep 17 00:00:00 2001 From: oneRain Date: Tue, 27 Aug 2019 11:52:53 +0800 Subject: [PATCH] =?UTF-8?q?*=20FileTest.cs:=20chore:=20=E5=AE=8C=E5=96=84?= =?UTF-8?q?=20File=20=E6=A8=A1=E5=9D=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Utils.cs: * AVFile.cs: * AWSUploader.cs: * QiniuUploader.cs: --- Storage/Storage.Test/FileTest.cs | 31 ++++++ Storage/Storage.Test/Utils.cs | 34 ++++++ .../Internal/File/Controller/AWSUploader.cs | 13 +-- .../Internal/File/Controller/QiniuUploader.cs | 104 +++++++----------- Storage/Storage/Public/AVFile.cs | 2 +- 5 files changed, 108 insertions(+), 76 deletions(-) create mode 100644 Storage/Storage.Test/FileTest.cs create mode 100644 Storage/Storage.Test/Utils.cs diff --git a/Storage/Storage.Test/FileTest.cs b/Storage/Storage.Test/FileTest.cs new file mode 100644 index 0000000..1b47b4c --- /dev/null +++ b/Storage/Storage.Test/FileTest.cs @@ -0,0 +1,31 @@ +using NUnit.Framework; +using LeanCloud; +using System.IO; +using System.Threading.Tasks; + +namespace LeanCloudTests { + public class FileTest { + [SetUp] + public void SetUp() { + //Utils.InitNorthChina(); + Utils.InitEastChina(); + //Utils.InitUS(); + } + + [Test] + public async Task SaveImage() { + AVFile file = new AVFile("hello.png", File.ReadAllBytes("hello.png")); + await file.SaveAsync(); + Assert.NotNull(file.ObjectId); + TestContext.Out.WriteLine($"file: {file.ObjectId}, {file.Url}"); + } + + [Test] + public async Task SaveBigFile() { + AVFile file = new AVFile("test.apk", File.ReadAllBytes("test.apk")); + await file.SaveAsync(); + Assert.NotNull(file.ObjectId); + TestContext.Out.WriteLine($"file: {file.ObjectId}, {file.Url}"); + } + } +} diff --git a/Storage/Storage.Test/Utils.cs b/Storage/Storage.Test/Utils.cs new file mode 100644 index 0000000..1c22b33 --- /dev/null +++ b/Storage/Storage.Test/Utils.cs @@ -0,0 +1,34 @@ +using System; +using LeanCloud; +using NUnit.Framework; + +namespace LeanCloudTests { + public static class Utils { + public static void InitNorthChina() { + AVClient.Initialize(new AVClient.Configuration { + ApplicationId = "BMYV4RKSTwo8WSqt8q9ezcWF-gzGzoHsz", + ApplicationKey = "pbf6Nk5seyjilexdpyrPwjSp", + ApiServer = "https://avoscloud.com" + }); + AVClient.HttpLog(TestContext.Out.WriteLine); + } + + public static void InitEastChina() { + AVClient.Initialize(new AVClient.Configuration { + ApplicationId = "4eTwHdYhMaNBUpl1SrTr7GLC-9Nh9j0Va", + ApplicationKey = "GSD6DtdgGWlWolivN4qhWtlE", + ApiServer = "https://4eTwHdYh.api.lncldapi.com" + }); + AVClient.HttpLog(TestContext.Out.WriteLine); + } + + public static void InitUS() { + AVClient.Initialize(new AVClient.Configuration { + ApplicationId = "MFAS1GnOyomRLSQYRaxdgdPz-MdYXbMMI", + ApplicationKey = "p42JUxdxb95K5G8187t5ba3l", + ApiServer = "https://MFAS1GnO.api.lncldglobal.com" + }); + AVClient.HttpLog(TestContext.Out.WriteLine); + } + } +} diff --git a/Storage/Storage/Internal/File/Controller/AWSUploader.cs b/Storage/Storage/Internal/File/Controller/AWSUploader.cs index 9a9413d..7cad558 100644 --- a/Storage/Storage/Internal/File/Controller/AWSUploader.cs +++ b/Storage/Storage/Internal/File/Controller/AWSUploader.cs @@ -4,6 +4,7 @@ using System.Threading; using System.IO; using System.Collections.Generic; using System.Net.Http; +using System.Net.Http.Headers; namespace LeanCloud.Storage.Internal { internal class AWSUploader : IFileUploader { @@ -19,21 +20,15 @@ namespace LeanCloud.Storage.Internal { } internal async Task PutFile(FileState state, string uploadUrl, Stream dataStream) { - IList> makeBlockHeaders = new List> { - new KeyValuePair("Content-Type", state.MimeType), - new KeyValuePair("Cache-Control", "public, max-age=31536000"), - new KeyValuePair("Content-Length", dataStream.Length.ToString()) - }; - HttpClient client = new HttpClient(); HttpRequestMessage request = new HttpRequestMessage { RequestUri = new Uri(uploadUrl), Method = HttpMethod.Put, Content = new StreamContent(dataStream) }; - foreach (var header in makeBlockHeaders) { - request.Headers.Add(header.Key, header.Value); - } + request.Headers.Add("Cache-Control", "public, max-age=31536000"); + request.Content.Headers.ContentType = MediaTypeHeaderValue.Parse(state.MimeType); + request.Content.Headers.ContentLength = dataStream.Length; await client.SendAsync(request); client.Dispose(); request.Dispose(); diff --git a/Storage/Storage/Internal/File/Controller/QiniuUploader.cs b/Storage/Storage/Internal/File/Controller/QiniuUploader.cs index c998daa..ec154e6 100644 --- a/Storage/Storage/Internal/File/Controller/QiniuUploader.cs +++ b/Storage/Storage/Internal/File/Controller/QiniuUploader.cs @@ -1,5 +1,4 @@ -using LeanCloud.Storage.Internal; -using System; +using System; using System.Collections.Generic; using System.IO; using System.Net; @@ -7,6 +6,7 @@ using System.Text; using System.Threading; using System.Threading.Tasks; using System.Net.Http; +using System.Net.Http.Headers; using Newtonsoft.Json; namespace LeanCloud.Storage.Internal { @@ -18,14 +18,9 @@ namespace LeanCloud.Storage.Internal { } internal class QiniuUploader : IFileUploader { - private static int BLOCKSIZE = 1024 * 1024 * 4; - private const int blockMashk = (1 << blockBits) - 1; - private const int blockBits = 22; - private int CalcBlockCount(long fsize) { - return (int)((fsize + blockMashk) >> blockBits); - } + private static readonly int BLOCKSIZE = 1024 * 1024 * 4; internal static string UP_HOST = "https://up.qbox.me"; - private object mutex = new object(); + private readonly object mutex = new object(); public Task Upload(FileState state, Stream dataStream, IDictionary fileToken, IProgress progress, CancellationToken cancellationToken) { state.frozenData = dataStream; @@ -49,13 +44,13 @@ namespace LeanCloud.Storage.Internal { } if (state.completed == totalSize) { return QiniuMakeFile(state, state.frozenData, state.token, state.CloudName, totalSize, state.block_ctxes.ToArray(), CancellationToken.None); - - } else if (state.completed % BLOCKSIZE == 0) { + } + if (state.completed % BLOCKSIZE == 0) { var firstChunkBinary = GetChunkBinary(state.completed, dataStream); var blockSize = remainingSize > BLOCKSIZE ? BLOCKSIZE : remainingSize; - return MakeBlock(state, firstChunkBinary, blockSize).ContinueWith(t => { - var dict = JsonConvert.DeserializeObject>(t.Result.Item2, new LeanCloudJsonConverter()); + return MakeBlock(state, firstChunkBinary, blockSize).OnSuccess(t => { + var dict = t.Result; var ctx = dict["ctx"].ToString(); offset = long.Parse(dict["offset"].ToString()); var host = dict["host"].ToString(); @@ -65,29 +60,23 @@ namespace LeanCloud.Storage.Internal { state.block_ctxes.Add(ctx); } - return UploadNextChunk(state, dataStream, ctx, offset, progress); - }).Unwrap(); - - } else { - var chunkBinary = GetChunkBinary(state.completed, dataStream); - return PutChunk(state, chunkBinary, context, offset).ContinueWith(t => { - var dict = JsonConvert.DeserializeObject>(t.Result.Item2, new LeanCloudJsonConverter()); - var ctx = dict["ctx"].ToString(); - - offset = long.Parse(dict["offset"].ToString()); - var host = dict["host"].ToString(); - state.completed += chunkBinary.Length; - if (state.completed % BLOCKSIZE == 0 || state.completed == totalSize) { - state.block_ctxes.Add(ctx); - } - //if (AVClient.fileUploaderDebugLog) - //{ - // AVClient.LogTracker(state.counter + "|completed=" + state.completed + "stream:position=" + dataStream.Position + "|"); - //} - return UploadNextChunk(state, dataStream, ctx, offset, progress); }).Unwrap(); } + var chunkBinary = GetChunkBinary(state.completed, dataStream); + return PutChunk(state, chunkBinary, context, offset).OnSuccess(t => { + var dict = t.Result; + var ctx = dict["ctx"].ToString(); + + offset = long.Parse(dict["offset"].ToString()); + var host = dict["host"].ToString(); + state.completed += chunkBinary.Length; + if (state.completed % BLOCKSIZE == 0 || state.completed == totalSize) { + state.block_ctxes.Add(ctx); + } + + return UploadNextChunk(state, dataStream, ctx, offset, progress); + }).Unwrap(); } byte[] GetChunkBinary(long completed, Stream dataStream) { @@ -101,56 +90,37 @@ namespace LeanCloud.Storage.Internal { return chunkBinary; } - internal Task>> GetQiniuToken(FileState state, CancellationToken cancellationToken) { - string str = state.Name; - - IDictionary parameters = new Dictionary(); - parameters.Add("name", str); - parameters.Add("key", state.CloudName); - parameters.Add("__type", "File"); - parameters.Add("mime_type", AVFile.GetMIMEType(str)); - - state.MetaData = GetMetaData(state, state.frozenData); - - parameters.Add("metaData", state.MetaData); - - var command = new AVCommand { - Path = "qiniu", - Method = HttpMethod.Post, - Content = parameters - }; - return AVPlugins.Instance.CommandRunner.RunCommandAsync>(command); - } IList> GetQiniuRequestHeaders(FileState state) { IList> makeBlockHeaders = new List>(); - string authHead = "UpToken " + state.token; makeBlockHeaders.Add(new KeyValuePair("Authorization", authHead)); return makeBlockHeaders; } - async Task> MakeBlock(FileState state, byte[] firstChunkBinary, long blcokSize = 4194304) { + async Task> MakeBlock(FileState state, byte[] firstChunkBinary, long blcokSize = 4194304) { MemoryStream firstChunkData = new MemoryStream(firstChunkBinary, 0, firstChunkBinary.Length); - var headers = GetQiniuRequestHeaders(state); - headers.Add(new KeyValuePair("Content-Type", "application/octet-stream")); + var client = new HttpClient(); var request = new HttpRequestMessage { RequestUri = new Uri($"{UP_HOST}/mkblk/{blcokSize}"), Method = HttpMethod.Post, Content = new StreamContent(firstChunkData) }; + var headers = GetQiniuRequestHeaders(state); foreach (var header in headers) { request.Headers.Add(header.Key, header.Value); } + + request.Content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); var response = await client.SendAsync(request); client.Dispose(); request.Dispose(); var content = await response.Content.ReadAsStringAsync(); response.Dispose(); - return await JsonUtils.DeserializeObjectAsync>(content); + return await JsonUtils.DeserializeObjectAsync>(content, new LeanCloudJsonConverter()); } - async Task> PutChunk(FileState state, byte[] chunkBinary, string LastChunkctx, long currentChunkOffsetInBlock) { + async Task> PutChunk(FileState state, byte[] chunkBinary, string LastChunkctx, long currentChunkOffsetInBlock) { MemoryStream chunkData = new MemoryStream(chunkBinary, 0, chunkBinary.Length); var client = new HttpClient(); var request = new HttpRequestMessage { @@ -167,7 +137,7 @@ namespace LeanCloud.Storage.Internal { request.Dispose(); var content = await response.Content.ReadAsStringAsync(); response.Dispose(); - return await JsonUtils.DeserializeObjectAsync>(content); + return await JsonUtils.DeserializeObjectAsync>(content); } internal async Task> QiniuMakeFile(FileState state, Stream dataStream, string upToken, string key, long fsize, string[] ctxes, CancellationToken cancellationToken) { @@ -184,12 +154,6 @@ namespace LeanCloud.Storage.Internal { } urlBuilder.Append(sb.ToString()); - IList> headers = new List>(); - //makeBlockDic.Add("Content-Type", "application/octet-stream"); - - string authHead = "UpToken " + upToken; - headers.Add(new KeyValuePair("Authorization", authHead)); - headers.Add(new KeyValuePair("Content-Type", "text/plain")); int proCount = ctxes.Length; Stream body = new MemoryStream(); @@ -208,6 +172,9 @@ namespace LeanCloud.Storage.Internal { Method = HttpMethod.Post, Content = new StreamContent(body) }; + request.Headers.Add("Authorization", $"UpToken {upToken}"); + request.Content.Headers.ContentType = MediaTypeHeaderValue.Parse("text/plain"); + var response = await client.SendAsync(request); client.Dispose(); request.Dispose(); @@ -232,6 +199,7 @@ namespace LeanCloud.Storage.Internal { return elements[elements.Length - 1]; } + public static byte[] StringToAscii(string s) { byte[] retval = new byte[s.Length]; for (int ix = 0; ix < s.Length; ++ix) { @@ -243,9 +211,11 @@ namespace LeanCloud.Storage.Internal { } return retval; } + public static string ToBase64URLSafe(string str) { return Encode(str); } + public static string Encode(byte[] bs) { if (bs == null || bs.Length == 0) return ""; @@ -253,6 +223,7 @@ namespace LeanCloud.Storage.Internal { encodedStr = encodedStr.Replace('+', '-').Replace('/', '_'); return encodedStr; } + public static string Encode(string text) { if (String.IsNullOrEmpty(text)) return ""; @@ -291,6 +262,7 @@ namespace LeanCloud.Storage.Internal { return rtn; } + internal void MergeDic(IDictionary dic, string key, object value) { if (dic.ContainsKey(key)) { dic[key] = value; diff --git a/Storage/Storage/Public/AVFile.cs b/Storage/Storage/Public/AVFile.cs index 3828f1d..d2f4cbe 100644 --- a/Storage/Storage/Public/AVFile.cs +++ b/Storage/Storage/Public/AVFile.cs @@ -300,7 +300,7 @@ namespace LeanCloud { return this.SaveExternal(); return taskQueue.Enqueue( - toAwait => FileController.SaveAsync(state, dataStream, AVUser.CurrentUser.SessionToken, progress, cancellationToken), cancellationToken) + toAwait => FileController.SaveAsync(state, dataStream, AVUser.CurrentUser?.SessionToken, progress, cancellationToken), cancellationToken) .OnSuccess(t => { state = t.Result; });