Need to see if your shared folder is taking up space on your dropbox 👨💻? Find out how to check here.
Forum Discussion
Jan B.23
11 years agoNew member | Level 1
unreliable data transfer w/ API
Hi,
I'm using API to transfer data from my computer to Dropbox. For 50 500MB files 5 have failed with the error like this:
Uploading "/var/lib/condor/execute/dir_2220/oneA.e15.oswrk136_500GB.da...
Jan B.23
11 years agoNew member | Level 1
What’s the actual error you're getting when making the failed calls?
The stderr reports for last 3 chunks of this file:
+ CHUNK_PARAMS='upload_id=0B3fybsi14V1MmpZ18As_Q&offset=209715200'
++ utime
+++ date +%s
++ echo 1434394232
+ curl -s --show-error --globoff -i -o /tmp/du_resp_debug --upload-file /tmp/du_chunk_1035 'https://api-content.dropbox.com/1/chunked_upload?upload_id=0B3fybsi14V1MmpZ18As_Q&offset=209715200&oauth_consumer_key=xeg3vxakd86svsh&oauth_token=jm624faonumhhxl9&oauth_signature_method=PLAINTEXT&oauth_signature=<redacted>%<redacted>&oauth_timestamp=1434394232&oauth_nonce=14453'
+ check_http_response
+ CODE=0
+ case $CODE in
+ grep -q 'HTTP/1.1 400' /tmp/du_resp_debug
+ grep -q '^HTTP/1.1 200 OK' /tmp/du_resp_debug
+ print '*'
+ [[ 0 == 0 ]]
+ echo -ne '*'
+ let UPLOAD_ERROR=0+1
+ [[ 1 -gt 2 ]]
+ [[ 209715200 != 524288000 ]]
+ let OFFSET_MB=209715200/1024/1024
+ dd if=/var/lib/condor/execute/dir_2220/oneA.e15.oswrk136_500GB.data of=/tmp/du_chunk_1035 bs=1048576 skip=200 count=100
+ [[ 209715200 != 0 ]]
+ CHUNK_PARAMS='upload_id=0B3fybsi14V1MmpZ18As_Q&offset=209715200'
++ utime
+++ date +%s
++ echo 1434394296
+ curl -s --show-error --globoff -i -o /tmp/du_resp_debug --upload-file /tmp/du_chunk_1035 'https://api-content.dropbox.com/1/chunked_upload?upload_id=0B3fybsi14V1MmpZ18As_Q&offset=209715200&oauth_consumer_key=xeg3vxakd86svsh&oauth_token=jm624faonumhhxl9&oauth_signature_method=PLAINTEXT&oauth_signature=<redacted>%<redacted>&oauth_timestamp=1434394296&oauth_nonce=5737'
+ check_http_response
+ CODE=0
+ case $CODE in
+ grep -q 'HTTP/1.1 400' /tmp/du_resp_debug
++ sed -n -e 's/{"error": "\([^"]*\)"}/\1/p' /tmp/du_resp_debug
+ ERROR_MSG=
+ case $ERROR_MSG in
+ grep -q '^HTTP/1.1 200 OK' /tmp/du_resp_debug
+ print '*'
+ [[ 0 == 0 ]]
+ echo -ne '*'
+ let UPLOAD_ERROR=1+1
+ [[ 2 -gt 2 ]]
+ [[ 209715200 != 524288000 ]]
+ let OFFSET_MB=209715200/1024/1024
+ dd if=/var/lib/condor/execute/dir_2220/oneA.e15.oswrk136_500GB.data of=/tmp/du_chunk_1035 bs=1048576 skip=200 count=100
+ [[ 209715200 != 0 ]]
+ CHUNK_PARAMS='upload_id=0B3fybsi14V1MmpZ18As_Q&offset=209715200'
++ utime
+++ date +%s
++ echo 1434394336
+ curl -s --show-error --globoff -i -o /tmp/du_resp_debug --upload-file /tmp/du_chunk_1035 'https://api-content.dropbox.com/1/chunked_upload?upload_id=0B3fybsi14V1MmpZ18As_Q&offset=209715200&oauth_consumer_key=xeg3vxakd86svsh&oauth_token=jm624faonumhhxl9&oauth_signature_method=PLAINTEXT&oauth_signature=<redacted>%<redacted>&oauth_timestamp=1434394336&oauth_nonce=21291'
+ check_http_response
+ CODE=0
+ case $CODE in
+ grep -q 'HTTP/1.1 400' /tmp/du_resp_debug
++ sed -n -e 's/{"error": "\([^"]*\)"}/\1/p' /tmp/du_resp_debug
+ ERROR_MSG=
+ case $ERROR_MSG in
+ grep -q '^HTTP/1.1 200 OK' /tmp/du_resp_debug
+ print '*'
+ [[ 0 == 0 ]]
+ echo -ne '*'
+ let UPLOAD_ERROR=2+1
+ [[ 3 -gt 2 ]]
+ print ' FAILED\n'
+ [[ 0 == 0 ]]
+ echo -ne ' FAILED\n'
+ print 'An error occurred requesting /chunked_upload\n'
+ [[ 0 == 0 ]]
+ echo -ne 'An error occurred requesting /chunked_upload\n'
+ ERROR_STATUS=1
+ return
+ (( i++ ))
+ (( i<6 ))
+ remove_temp_files
+ [[ 1 == 0 ]]
+ exit 1
====================================================================================
====================================================================================
What chunk size are you using?
CHUNK_SIZE=100
It is in MB. The code sending data looks like this:
#Chunked file upload
#$1 = Local source file
#$2 = Remote destination file
function db_chunked_upload_file
{
local FILE_SRC=$(normalize_path "$1")
local FILE_DST=$(normalize_path "$2")
print " > Uploading \"$FILE_SRC\" to \"$FILE_DST\"
local FILE_SIZE=$(file_size "$FILE_SRC")
local OFFSET=0
local UPLOAD_ID="
local UPLOAD_ERROR=0
local CHUNK_PARAMS="
#Uploading chunks...
while ([[ $OFFSET != $FILE_SIZE ]]); do
let OFFSET_MB=$OFFSET/1024/1024
#Create the chunk
dd if="$FILE_SRC" of="$CHUNK_FILE" bs=1048576 skip=$OFFSET_MB count=$CHUNK_SIZE 2> /dev/null
#Only for the first request these parameters are not included
if [[ $OFFSET != 0 ]]; then
CHUNK_PARAMS="upload_id=$UPLOAD_ID&offset=$OFFSET"
fi
#Uploading the chunk...
$CURL_BIN $CURL_ACCEPT_CERTIFICATES -s --show-error --globoff -i -o "$RESPONSE_FILE" --upload-file "$CHUNK_FILE" "$API_CHUNKED_UPLOAD_URL?$CHUNK_PARAMS&oauth_consumer_key=$APPKEY&oauth_token=$OAUTH_ACCESS_TOKEN&oauth_signature_method=PLAINTEXT&oauth_signature=$APPSECRET%26$OAUTH_ACCESS_TOKEN_SECRET&oauth_timestamp=$(utime)&oauth_nonce=$RANDOM" 2> /dev/null
check_http_response
#Check
if grep -q "^HTTP/1.1 200 OK" "$RESPONSE_FILE"; then
print "."
UPLOAD_ERROR=0
UPLOAD_ID=$(sed -n 's/.*"upload_id": *"*\([^"]*\)"*.*/\1/p' "$RESPONSE_FILE")
OFFSET=$(sed -n 's/.*"offset": *\([^}]*\).*/\1/p' "$RESPONSE_FILE")
else
print "*"
let UPLOAD_ERROR=$UPLOAD_ERROR+1
#On error, the upload is retried for max 3 times
if [[ $UPLOAD_ERROR -gt 2 ]]; then
print " FAILED\n"
print "An error occurred requesting /chunked_upload\n"
ERROR_STATUS=1
return
fi
fi
done
UPLOAD_ERROR=0
#Commit the upload
while (true); do
$CURL_BIN $CURL_ACCEPT_CERTIFICATES -s --show-error --globoff -i -o "$RESPONSE_FILE" --data "upload_id=$UPLOAD_ID&oauth_consumer_key=$APPKEY&oauth_token=$OAUTH_ACCESS_TOKEN&oauth_signature_method=PLAINTEXT&oauth_signature=$APPSECRET%26$OAUTH_ACCESS_TOKEN_SECRET&oauth_timestamp=$(utime)&oauth_nonce=$RANDOM" "$API_CHUNKED_UPLOAD_COMMIT_URL/$ACCESS_LEVEL/$(urlencode "$FILE_DST")" 2> /dev/null
check_http_response
#Check
if grep -q "^HTTP/1.1 200 OK" "$RESPONSE_FILE"; then
print "."
UPLOAD_ERROR=0
break
else
print "*"
let UPLOAD_ERROR=$UPLOAD_ERROR+1
#On error, the commit is retried for max 3 times
if [[ $UPLOAD_ERROR -gt 2 ]]; then
print " FAILED\n"
print "An error occurred requesting /commit_chunked_upload\n"
ERROR_STATUS=1
return
fi
fi
done
print " DONE\n"
}
About Dropbox API Support & Feedback
Find help with the Dropbox API from other developers.
The Dropbox Community team is active from Monday to Friday. We try to respond to you as soon as we can, usually within 2 hours.
If you need more help you can view your support options (expected response time for an email or ticket is 24 hours), or contact us on X, Facebook or Instagram.
For more info on available support options for your Dropbox plan, see this article.
If you found the answer to your question in this Community thread, please 'like' the post to say thanks and to let us know it was useful!