moving uploads to streams (fixing #2) · 537b02da63 - SVN.BY: Go Git Service
Просмотр исходного кода

moving uploads to streams (fixing #2)

Tiernan OToole лет назад: 8
Родитель
Сommit
537b02da63
2 измененных файлов с 30 добавлено и 33 удалено
  1. 1 1
      B2Uploader/B2Uploader.csproj
  2. 29 32
      B2Uploader/Program.cs

+ 1 - 1
B2Uploader/B2Uploader.csproj

17
     <RestorePackages>true</RestorePackages>
17
     <RestorePackages>true</RestorePackages>
18
   </PropertyGroup>
18
   </PropertyGroup>
19
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
19
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
20
-    <PlatformTarget>AnyCPU</PlatformTarget>
20
+    <PlatformTarget>x64</PlatformTarget>
21
     <DebugSymbols>true</DebugSymbols>
21
     <DebugSymbols>true</DebugSymbols>
22
     <DebugType>full</DebugType>
22
     <DebugType>full</DebugType>
23
     <Optimize>false</Optimize>
23
     <Optimize>false</Optimize>

+ 29 - 32
B2Uploader/Program.cs

46
                 var bucket = buckets.buckets.First();
46
                 var bucket = buckets.buckets.First();
47
 
47
 
48
                 string[] FilesToProcess = Directory.GetFiles(options.Directory);
48
                 string[] FilesToProcess = Directory.GetFiles(options.Directory);
49
-
50
-                Parallel.ForEach(FilesToProcess, s =>
49
+                Parallel.ForEach(FilesToProcess, new ParallelOptions() { MaxDegreeOfParallelism = 32 }, s =>
51
                 {
50
                 {
52
                     //check if file already exists
51
                     //check if file already exists
53
 
52
 
108
         {
107
         {
109
             var headers = GetAuthHeaders(authToken);
108
             var headers = GetAuthHeaders(authToken);
110
 
109
 
111
-            string responseString = MakeWebRequest(apiUrl + "/b2api/v1/b2_list_buckets", headers, request);
110
+            string responseString = MakeRequest(apiUrl + "/b2api/v1/b2_list_buckets", headers, JsonConvert.SerializeObject(request));
112
 
111
 
113
 
112
 
114
             return JsonConvert.DeserializeObject<ListBucketsResponse>(responseString);
113
             return JsonConvert.DeserializeObject<ListBucketsResponse>(responseString);
125
         {
124
         {
126
 
125
 
127
             var headers = GetAuthHeaders(authToken); 
126
             var headers = GetAuthHeaders(authToken); 
128
-            string responseString = MakeWebRequest(apiUrl + "/b2api/v1/b2_get_upload_url", headers, request);
127
+            string responseString = MakeRequest(apiUrl + "/b2api/v1/b2_get_upload_url", headers, JsonConvert.SerializeObject(request));
129
             
128
             
130
             return JsonConvert.DeserializeObject<GetUploadURLResponse>(responseString);
129
             return JsonConvert.DeserializeObject<GetUploadURLResponse>(responseString);
131
         }
130
         }
141
 
140
 
142
         static UploadFileResponse UploadFile(string authToken, string contentType, string filePath, string uploadUrl)
141
         static UploadFileResponse UploadFile(string authToken, string contentType, string filePath, string uploadUrl)
143
         {
142
         {
144
-
145
-            byte[] bytes = System.IO.File.ReadAllBytes(filePath);
146
-            String sha1 = GetSha1(bytes);
143
+            FileStream fs = System.IO.File.OpenRead(filePath);
144
+            String sha1 = GetSha1(fs);
147
 
145
 
148
             var headers = GetAuthHeaders(authToken);
146
             var headers = GetAuthHeaders(authToken);
149
 
147
 
152
             headers.Add(new Tuple<string, string>("X-Bz-File-Name", fileName));
150
             headers.Add(new Tuple<string, string>("X-Bz-File-Name", fileName));
153
             headers.Add(new Tuple<string, string>("X-Bz-Content-Sha1", sha1));
151
             headers.Add(new Tuple<string, string>("X-Bz-Content-Sha1", sha1));
154
 
152
 
155
-            string responseString = MakeWebRequest(uploadUrl, headers, bytes, contentType);
153
+            string responseString = MakeRequest(uploadUrl, headers, fs, contentType);
156
 
154
 
157
             var resp = JsonConvert.DeserializeObject<UploadFileResponse>(responseString);
155
             var resp = JsonConvert.DeserializeObject<UploadFileResponse>(responseString);
158
 
156
 
171
         static ListFileNamesResponse ListFileNames(ListFileNamesRequest request, string apiUrl, string authToken)
169
         static ListFileNamesResponse ListFileNames(ListFileNamesRequest request, string apiUrl, string authToken)
172
         {
170
         {
173
             var headers = GetAuthHeaders(authToken);
171
             var headers = GetAuthHeaders(authToken);
174
-            string responseString =  MakeWebRequest(apiUrl + "/b2api/v1/b2_list_file_names", headers, request);
172
+            string responseString =  MakeRequest(apiUrl + "/b2api/v1/b2_list_file_names", headers, JsonConvert.SerializeObject(request));
175
 
173
 
176
             return JsonConvert.DeserializeObject<ListFileNamesResponse>(responseString);
174
             return JsonConvert.DeserializeObject<ListFileNamesResponse>(responseString);
177
         }
175
         }
178
 
176
 
179
-        static string MakeWebRequest<T>(string url, List<Tuple<string,string>> headers, T item, string contentType = "application/json; charset=utf-8")
177
+
178
+        static string MakeRequest(string url, List<Tuple<string,string>> headers, string data, string contentType = "application/json; charset=urf-8")
180
         {
179
         {
181
-            string body = string.Empty;
182
-             byte[] data;
180
+            MemoryStream ms = new MemoryStream(Encoding.UTF8.GetBytes(data));
181
+            return MakeRequest(url, headers, ms, contentType);
182
+        }
183
+        
184
+        static string MakeRequest(string url, List<Tuple<string,string>> headers, Stream data, string contentType="application/json; charset=utf-8")
185
+        {            
183
             try
186
             try
184
             {
187
             {
185
                 HttpWebRequest req = (HttpWebRequest)WebRequest.Create(url);
188
                 HttpWebRequest req = (HttpWebRequest)WebRequest.Create(url);
186
-               
187
-
188
-                if (typeof(T) == typeof(byte[]))
189
-                {
190
-                    data = (byte[])(object)item;
191
-                }
192
-                else
193
-                {
194
-                    body =  JsonConvert.SerializeObject(item);
195
-
196
-                    data = Encoding.UTF8.GetBytes(body);
197
-                }
198
 
189
 
199
                 req.Method = "POST";
190
                 req.Method = "POST";
200
 
191
 
203
                     req.Headers.Add(head.Item1, head.Item2);
194
                     req.Headers.Add(head.Item1, head.Item2);
204
                 }
195
                 }
205
 
196
 
206
-                req.ContentType = contentType;
207
-                req.ContentLength = data.Length;
208
                 using (var stream = req.GetRequestStream())
197
                 using (var stream = req.GetRequestStream())
209
                 {
198
                 {
210
-                    stream.Write(data, 0, data.Length);
199
+                    
200
+                    data.Position = 0;
201
+
202
+                    req.ContentType = contentType;
203
+                    
204
+                    data.CopyTo(stream);
205
+                    data.Flush();
206
+                    
211
                     stream.Close();
207
                     stream.Close();
212
                 }
208
                 }
213
                 WebResponse response = (HttpWebResponse)req.GetResponse();
209
                 WebResponse response = (HttpWebResponse)req.GetResponse();
220
             {
216
             {
221
                 Console.WriteLine("Error talking to server: {0}", ex.Message);
217
                 Console.WriteLine("Error talking to server: {0}", ex.Message);
222
                 Console.WriteLine("URL: {0}", url);
218
                 Console.WriteLine("URL: {0}", url);
223
-                Console.WriteLine("Body: {0}", body);
224
                 throw;
219
                 throw;
225
             }
220
             }
226
-
227
-
228
         }
221
         }
229
 
222
 
230
-        private static string GetSha1(byte[] bytes)
223
+        
224
+        
225
+        
226
+
227
+        private static string GetSha1(Stream inputStream)
231
         {
228
         {
232
             using (SHA1Managed sha1 = new SHA1Managed())
229
             using (SHA1Managed sha1 = new SHA1Managed())
233
             {
230
             {
234
-                var hash = sha1.ComputeHash(bytes);
231
+                var hash = sha1.ComputeHash(inputStream);
235
                 var sb = new StringBuilder(hash.Length * 2);
232
                 var sb = new StringBuilder(hash.Length * 2);
236
 
233
 
237
                 foreach (byte b in hash)
234
                 foreach (byte b in hash)