File size increased if resuming ChunkedUpload after program exits.

[0ylanmorisson]

I was testing this ChunkedUpload code and manually making the program exit and restart several times to test if chunkedupload worked.

The code logic is this: after finishing each successful uploading loop, the program writes down the sessionID, the uploading offset and the index to a log.txt  in case that the program accidentally exits.

So when restarting the program, it will reload the offset and index and sessionID from log.txt to resume last failed upload. And if the recorded offset was wrong, it retrieves the correct offset from dropbox sever and uploads from the correct position of the file.

The code works fine if there is no interruptions of the uploading process. But once the program exits and restarts, even if it retrieves the correct offset from dropbox server, it still uploads more bytes to the file and makes the final-uploaded-file-size increased and of course the file was corrupted.

Here is the code.

        private async Task ChunkedUpload(DropboxClient client, string srcfile, string dstfile)
        {
            if (!File.Exists(srcfile))
            {
                Environment.Exit(0);
            }

            // Chunk size is 1024 * 1024 = 1048576B.
            int chunkSize = 1000000;
            long currentPosition = 0;

            FileStream fs = new FileStream(srcfile, FileMode.Open);
            byte[] bytesToBeEncrypted = new byte[fs.Length];
            fs.Read(bytesToBeEncrypted, 0, bytesToBeEncrypted.Length);
            fs.Close();

            using (var stream = new MemoryStream(bytesToBeEncrypted))
            {
                int numChunks = (int)Math.Ceiling((double)stream.Length / chunkSize);
                var idx = 0;
                if ( numChunks < 2 )
                {
                    Console.WriteLine("Chunk size too big, using one time upload.");
                    try
                    {
                        var response = await client.Files.UploadAsync(dstfile, WriteMode.Overwrite.Instance, body: stream);
                        Console.WriteLine("Uploaded file to : {0} OK!\n", dstfile);
                    }
                    catch (Exception e)
                    {
                        Console.WriteLine("Error uploading {0}. Error: {1}\n", srcfile, e.Message);
                    }
                    return;
                }

                byte[] buffer = new byte[chunkSize];
                string sessionId = null;
                bool isUploaded = true;
                bool isResume = false;
                var byteRead=0;
                string logpath = this.GetType().Assembly.Location + ".log.txt"; // + @"\" 

                if (File.Exists(logpath))
                {
                    StreamReader objReader = new StreamReader(logpath);
                    isResume = true;
                    string line;
                    int nline = 0;
                    while ((line = objReader.ReadLine()) != null)
                    {
                        if ( nline ==0)
                        {
                            sessionId = line;
                            nline++;
                            continue;
                        }
                        if ( nline == 1)
                        {
                            currentPosition = long.Parse(line);
                            nline++;
                            continue;
                        }
                        if (nline == 2)
                        {
                            idx = int.Parse(line);
                            nline++;
                            continue;
                        }
                    }
                    Console.WriteLine("\nReloading log.. SessionID:{0}. Offset:{1}.\n", sessionId, currentPosition);
                    objReader.Close();
                }

                do
                {
                    if (isUploaded)
                    {
                        byteRead = stream.Read(buffer, 0, chunkSize);
                    }

                    if (isResume)
                    {
                        stream.Seek(currentPosition, SeekOrigin.Begin);
                        byteRead = stream.Read(buffer, 0, chunkSize);
                    }
                    try
                    {
                        using (MemoryStream memStream = new MemoryStream(buffer, 0, byteRead))
                        {
                            if (idx == 0)
                            {
                                var result = await client.Files.UploadSessionStartAsync(body: memStream);
                                sessionId = result.SessionId;
                                isUploaded = true;
                                isResume = false;
                                Console.WriteLine("\nStart uploading file {0} to Dropbox path: {1}.\n", srcfile, dstfile );
                            }
                            else
                            {
                                UploadSessionCursor cursor = new UploadSessionCursor(sessionId, (ulong)currentPosition);
                                if (idx == numChunks - 1)
                                {
                                    await client.Files.UploadSessionFinishAsync(cursor, new CommitInfo(dstfile), memStream);
                                    isUploaded = true;
                                    isResume = false;
                                }
                                else
                                {
                                    await client.Files.UploadSessionAppendV2Async(cursor, body: memStream);
                                    isUploaded = true;
                                    isResume = false;
                                }
                                FileStream log = new FileStream(logpath, FileMode.OpenOrCreate);
                                StreamWriter sw = new StreamWriter(log);
                                sw.WriteLine(sessionId.ToString());
                                sw.WriteLine(cursor.Offset.ToString());
                                sw.WriteLine(idx.ToString());
                                sw.Close();
                                Console.WriteLine("Uploaded {0}: {1}b of total {2}b. SessionID: {3}.", idx, cursor.Offset.ToString(), (int)Math.Ceiling((double)stream.Length), sessionId);
                            }
                         }
                    }
                    catch (ApiException<UploadSessionLookupError> e)
                    {
                        if (e.ErrorResponse.IsIncorrectOffset)
                        {
                            currentPosition = int.Parse(e.ErrorResponse.AsIncorrectOffset.Value.CorrectOffset.ToString());
                            StreamWriter sw = new StreamWriter(logpath);
                            sw.WriteLine(sessionId.ToString());
                            sw.WriteLine(e.ErrorResponse.AsIncorrectOffset.Value.CorrectOffset.ToString());
                            sw.WriteLine(idx.ToString());
                            sw.Close();
                            Console.WriteLine("\nError uploading:{0}. SessionID: {1}. CorrectOffset:{2}. Reuploading..\n", e.Message, sessionId, e.ErrorResponse.AsIncorrectOffset.Value.CorrectOffset.ToString());
                            currentPosition = int.Parse(e.ErrorResponse.AsIncorrectOffset.Value.CorrectOffset.ToString());
                            isUploaded = false;
                            isResume = false;
                            idx++;
                            continue;
                          }
                    }
                    catch (Exception e)
                    {
                        isUploaded = false;
                        isResume = false;
                        Console.WriteLine("\nError uploading:{0}. SessionID: {1}. Retrying...\n", e.Message, sessionId, sessionId );
                        //Environment.Exit(0);
                        continue;
                    }
                    idx++;
                    currentPosition = currentPosition + byteRead;
                } while (idx < numChunks);
                File.Delete(logpath);
            }
            Console.WriteLine("\nUpload OK! Byebye...");
        }

[Greg-DB]

It's difficult to say off hand where the issue may be. Have you tried stepping through with the debugger to see where/why your code is uploading more than expected?