Chunked upload with HTTP requests

My guess would be an encoding bug on your side. (Perhaps you're treating the data as text instead of binary?)

Can you share your code?

Written in scala, so far for testing purposes no other handling is done. Due to the need to have 100% e2e test coverage it was decided against using java SDK (mainly due to having trouble supplying local trusted-certs.jks, seemed easier overall to just do everything with plain HTTP requests)

for debugging purposes there are certain variables made just to see more data, but not used, please disregard that


def chunkedUpload(): Any = {

  val fileToUpload = new File(getClass.getClassLoader.getResource("files/app.psd").toURI)
  val defaultChunkSize: Int = 4 * 1024 * 1024
  val fileInByteArray = Files.readAllBytes(fileToUpload.toPath)
  val httpClient = HttpClients.createDefault()
  val length = fileInByteArray.length
  val chunkingCounter = fileInByteArray.length / defaultChunkSize + 1

  def getChunk(byteArray: Array[Byte], counter: Int, chunkSize: Int = defaultChunkSize): HttpEntity = {

    val byteArrayLength = byteArray.length

    if (counter == 0) {

      val multipartEntityBuilder = MultipartEntityBuilder.create()

      if (byteArrayLength <= chunkSize) {
        val chunk = byteArray
        val chunklength = chunk.length
        multipartEntityBuilder.addBinaryBody("chunk", chunk)
        val requestBody = multipartEntityBuilder.build()
        requestBody
      }
      else {
        val chunk = byteArray.slice(0, chunkSize)
        val chunklength = chunk.length
        multipartEntityBuilder.addBinaryBody("chunk", chunk)
        val requestBody = multipartEntityBuilder.build()
        requestBody
      }
    }
    else {
      val leftoverArray = byteArray.drop(chunkSize)
      val leftoverLength = leftoverArray.length
      getChunk(leftoverArray, counter - 1, chunkSize)
    }
  }

  def uploadChunk(requestBody: HttpEntity, uploadUrl: String): DBObject = {
    val uploadRequest = new HttpPut(uploadUrl)
    uploadRequest.setHeader("Authorization", s"Bearer $token")
    uploadRequest.setEntity(requestBody)
    val response = httpClient.execute(uploadRequest)
    val parsedResponse = JSON.parse(EntityUtils.toString(response.getEntity)).asInstanceOf[DBObject]
    parsedResponse
  }

  def commitChunkedUpload(uploadUrl: String, uploadId: String): Any = {
    val commitUploadRequest = new HttpPost(uploadUrl)
    commitUploadRequest.setHeader("Authorization", s"Bearer $token")
    val postParameters = new util.ArrayList[NameValuePair]()
    postParameters.add(new BasicNameValuePair("upload_id", uploadId))
    postParameters.add(new BasicNameValuePair("overwrite", "true"))
    commitUploadRequest.setEntity(new UrlEncodedFormEntity(postParameters))
    val commitResponse = httpClient.execute(commitUploadRequest)
    val parsedResponse = JSON.parse(EntityUtils.toString(commitResponse.getEntity)).asInstanceOf[DBObject]
    parsedResponse
  }

  def executeChunkedUpload(counter: Int, uploadId: String = ", offset: Long = 0): Any = {
    if (counter == chunkingCounter) commitChunkedUpload(
      s"https://content.dropboxapi.com/1/commit_chunked_upload/auto/Stuff/add%20pdf.psd", uploadId) //try setting parameters as post params instead of URL
    else {
      val requestBody = getChunk(fileInByteArray, counter)
      val uploadUrl = s"https://content.dropboxapi.com/1/chunked_upload?upload_id=$uploadId&offset=$offset"
      val intermediaryResponse = uploadChunk(requestBody, uploadUrl)
      val newOffset = intermediaryResponse.get("offset").asInstanceOf[Int]
      val rewriteId = intermediaryResponse.get("upload_id").asInstanceOf[String]
      executeChunkedUpload(counter + 1, rewriteId, newOffset)
    }
  }

  executeChunkedUpload(0)

}

It looks like you're sending multipart form data, which explains the extra data you're sending. You should instead be sending the raw bytes of the file as the body of the HTTP request.

From a quick search, I think ByteArrayEntity is what you want (instead of MultipartEntity).

Just switched to ByteArrayEntity as you suggested and everything works as intented!

Many thanks!