using B2Classes; using CommandLine; using Newtonsoft.Json; using NLog; using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Linq; using System.Net; using System.Net.Http; using System.Security.Cryptography; using System.Text; using System.Threading; using System.Threading.Tasks; namespace B2Uploader { class CmdLineOptions { [Option('i', "accountid", HelpText = "Account ID", Required=true)] public string AccountId { get; set; } [Option('a', "appkey", HelpText = "Application Key", Required=true)] public string ApplicationKey { get; set; } [Option('d', "directory", HelpText = "Directory you want to upload", Required = true)] public string Directory { get; set; } [Option('m', "multithreads", HelpText = "Number of uploads you want to use at a time. Default is 2", Required = false)] public int Threads { get; set; } [Option('r', "recursive", HelpText = "Uploads the directory in Recursive mode. Any sub folders will also be uploaded", Required = false)] public bool recursive { get; set; } [Option('v', "verbose", HelpText="Verbose Output")] public bool Verbose{get;set;} } class Program { static Logger logger = LogManager.GetCurrentClassLogger(); static void Main(string[] args) { var result = CommandLine.Parser.Default.ParseArguments(args); logger.Debug("Starting B2Uploader Version: {0}", System.Reflection.Assembly.GetExecutingAssembly().GetName().Version); var existCode = result.MapResult(options => { if (!Directory.Exists(options.Directory)) { Console.WriteLine("Directory to upload MUST EXIST!"); return 0; } if (options.Verbose) { logger.Debug("Authorizing User"); } var auth = AuthorizeUser(options.AccountId, options.ApplicationKey); if (options.Verbose) { logger.Debug("Listing Buckets"); } var buckets = ListBuckets(new ListBucketsRequest() { accountId = auth.accountId }, auth.authorizationToken, auth.apiUrl).Result; var bucket = buckets.buckets.First(); logger.Debug("Using Bucket Named {0} for uploads", bucket.bucketName); SearchOption so = SearchOption.TopDirectoryOnly; if (options.recursive) { if (options.Verbose) { logger.Debug("Using Reurisve mode for uploads"); } so = SearchOption.AllDirectories; } string[] FilesToProcess = Directory.GetFiles(options.Directory, "*", so); if (options.Verbose) { logger.Debug("Found {0} files to upload", FilesToProcess.Length); } int maxParallel = 2; if(options.Threads > 0) { if (options.Verbose) { logger.Debug("Mutliple threads for upload: {0}", options.Threads); } maxParallel = options.Threads; } Parallel.ForEach(FilesToProcess, new ParallelOptions() { MaxDegreeOfParallelism = maxParallel }, s => { //check if file already exists string fileName = getValidFilename(s); var existingFiles = ListFileNames(new ListFileNamesRequest() { bucketId = bucket.bucketId, startFileName = fileName }, auth.apiUrl, auth.authorizationToken); bool found = false; foreach (var x in existingFiles.files) { if (x.fileName == fileName) { //check the file size System.IO.FileInfo fi = new System.IO.FileInfo(s); if (fi.Length == x.size) { found = true; break; } else { //delete old file? could just be an older version... going to upload again... break; } } } if (found) { logger.Debug("File {0} exists already, skipping", fileName); } else { bool uploaded = false; int retries = 0; while (!uploaded && retries < 3) { try { logger.Debug("Getting Upload URL"); var uploadURL = GetUploadURL(new GetUploadURLRequest { bucketId = bucket.bucketId }, auth.apiUrl, auth.authorizationToken).Result; logger.Debug("Starting upload"); var response = UploadFile(uploadURL.authorizationToken, "b2/x-auto", s, uploadURL.uploadUrl); if(response != null) { uploaded = true; } } catch(Exception ex) { logger.Error("Uploaded Failed. {0}, Retrying",ex.Message); var innerEx = ex.InnerException; while(innerEx != null) { logger.Error("Inner Ex: {0}", innerEx.Message); innerEx = innerEx.InnerException; } logger.Error(ex.Message); uploaded = false; retries += 1; Thread.Sleep(TimeSpan.FromSeconds(30)); } } if (!uploaded) { logger.Error("Uploaded Failed 3 times... Please retry later!"); } } }); return 1; }, errors =>{ Console.WriteLine(errors); return 1; }); } static AuthorizeResponse AuthorizeUser(string accountId, string applicationKey) { HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create("https://api.backblaze.com/b2api/v1/b2_authorize_account"); string credentials = Convert.ToBase64String(Encoding.UTF8.GetBytes(string.Format("{0}:{1}", accountId, applicationKey))); webRequest.Headers.Add("Authorization", "Basic " + credentials); webRequest.ContentType = "application/json; charset=utf-8"; WebResponse response = (HttpWebResponse)webRequest.GetResponse(); var responseString = new StreamReader(response.GetResponseStream()).ReadToEnd(); response.Close(); return JsonConvert.DeserializeObject(responseString); } static async Task ListBuckets(ListBucketsRequest request, string authToken, string apiUrl) { var headers = GetAuthHeaders(authToken); string responseString = await MakeRequest2(apiUrl + "/b2api/v1/b2_list_buckets", headers, JsonConvert.SerializeObject(request)); return await Task.Factory.StartNew(() => JsonConvert.DeserializeObject(responseString)); } static List> GetAuthHeaders(string authToken) { List> headers = new List>(); headers.Add(new Tuple("Authorization", authToken)); return headers; } static async Task GetUploadURL(GetUploadURLRequest request, string apiUrl, string authToken) { var headers = GetAuthHeaders(authToken); string responseString = await MakeRequest2(apiUrl + "/b2api/v1/b2_get_upload_url", headers, JsonConvert.SerializeObject(request)); return await Task.Factory.StartNew(() => JsonConvert.DeserializeObject(responseString)); } static string getValidFilename(string input) { string fileName = input.Replace('\\', '/'); fileName = fileName.Replace(' ', '_'); if (fileName.StartsWith("/")) { fileName = fileName.Substring(1); } return fileName; } static UploadFileResponse UploadFile(string authToken, string contentType, string filePath, string uploadUrl) { logger.Debug("Starting Uploading {0}", filePath); String sha1 = GetSha1(filePath); var headers = GetAuthHeaders(authToken); string fileName = getValidFilename(filePath); headers.Add(new Tuple("X-Bz-File-Name", fileName)); headers.Add(new Tuple("X-Bz-Content-Sha1", sha1)); string responseString = MakeRequest2(uploadUrl, headers, filePath, true, contentType).Result; var resp = JsonConvert.DeserializeObject(responseString); if (resp.contentSha1 == sha1) { Console.WriteLine(responseString); return resp; } else { //something went wrong! return null; } } static ListFileNamesResponse ListFileNames(ListFileNamesRequest request, string apiUrl, string authToken) { var headers = GetAuthHeaders(authToken); string responseString = MakeRequest2(string.Format("{0}/b2api/v1/b2_list_file_names", apiUrl), headers, JsonConvert.SerializeObject(request)).Result; return JsonConvert.DeserializeObject(responseString); } static async Task MakeRequest2(string url, List> headers, string data, bool isFile = false, string contentType = "application/json; charset=utf-8") { var client = new HttpClient(); if (isFile) { client.Timeout = TimeSpan.FromMinutes(60); } foreach (var head in headers) { client.DefaultRequestHeaders.Add(head.Item1, head.Item2); } HttpContent content = null; if (isFile) { content = new StreamContent(System.IO.File.OpenRead(data)); content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue(contentType); } else { content = new StringContent(data); } var resp = await client.PostAsync(url, content); try { resp.EnsureSuccessStatusCode(); return await resp.Content.ReadAsStringAsync(); } catch(Exception ex) { logger.Error("Error getting respoonse: {0} {1}", ex.Message, ex.StackTrace); throw; } } private static string GetSha1(string fileName) { using (SHA1Managed sha1 = new SHA1Managed()) { using (FileStream fs = System.IO.File.OpenRead(fileName)) { var hash = sha1.ComputeHash(fs); var sb = new StringBuilder(hash.Length * 2); foreach (byte b in hash) { // can be "x2" if you want lowercase sb.Append(b.ToString("X2")); } return sb.ToString(); } } } } }