1
\$\begingroup\$

This is a follow on to a previous question:

Part 2: Upload/Download

All the code I am submitting here is available in github

The reason to write these tools is so I can prototype and quickly verify that I am doing things correctly before implementing them in C++. This is mainly necessary because the documentation these scripts are based on are relatively bad and confusing.

s3/upload

#!/usr/bin/env bash

function upload {
    local file=$1
    eval file=${file}
    local s3=$2
    local accessKey=$3
    local secretKey=$4
    local dateTime=$(echo $5 | xargs)
    local verbose=$6
    local dryrun=$7

    local s3Path=${s3#s3://}
    local bucket=${s3Path%%/*}
    local path=${s3Path#*/}

    if [[ "${s3}" == "${s3Path}" ]]; then
        echo "Error: destination ${s3} is not correct"
        echo "An s3 url looks like this:  s3://<bucket>/<object>"
        exit 1
    fi

    if [[ "${dateTime}" == "" ]]; then
        dateTime=$(date -u +"%Y%m%dT%H%M%SZ")
    fi

    local size=$(stat -f "%z" ${file})

    local url=https://${bucket}.s3.amazonaws.com/${path}
    declare -a headers=("Content-Length:${size}")

    verboseTmp=''
    if [[ "${verbose}" != "" ]]; then
        verboseTmp="/tmp/$$.put.tmp"
    fi

    local signature=$(createSignature PUT "${url}" "${accessKey}" "${secretKey}" ${dateTime} 3600 's3' 'us-east-1' 1 "${verboseTmp}" headers[@])

    if [[ "${verbose}" != "" ]]; then
        cat ${verboseTmp} > ${verbose}
        cat - <<IntermediateValuese > ${verbose}

File:           ${file}
S3:             ${s3}
Prt:            ${s3Path}
Bucket:         ${bucket}
Path:           ${path}

dateTime:       ${dateTime}
Size:           ${size}

URL:            ${url}
signature:      ${signature}


IntermediateValuese
    local id=$$
    if [[ ${dryrun} == 1 ]]; then
        id=dryrun
    fi
    local response="/tmp/${id}.response"
    echo                >> ${verbose}
    echo "UPLOAD"       >> ${verbose}
    echo "======"       >> ${verbose}
    echo "curl -o ${response} --dump-header ${response}.header --header \"${signature}\" --header \"Content-Length: ${size}\" --header \"X-Amz-Content-SHA256: UNSIGNED-PAYLOAD\" --header \"X-Amz-Date: ${dateTime}\" --data-binary \"@${file}\" -X PUT ${url}" >> ${verbose}
    fi

    if [[ ${dryrun} == 0 ]]; then
        curl -o ${response} --dump-header ${response}.header --header "${signature}" --header "Content-Length: ${size}" --header "X-Amz-Content-SHA256: UNSIGNED-PAYLOAD" --header "X-Amz-Date: ${dateTime}" --data-binary "@${file}" -X PUT ${url}
        if [[ "${verbose}" != "" ]]; then
            echo "HEADER"           >> ${verbose}
            echo "======"           >> ${verbose}
            cat ${respose}.header   >> ${verbose}
            echo "BODY"             >> ${verbose}
            echo "===="             >> ${verbose}
            cat ${respose}          >> ${verbose}
        fi
    fi
}

s3/downlaod

#!/usr/bin/env bash

function download {
    local s3=$1
    local file=$2
    eval file=${file}
    local accessKey=$3
    local secretKey=$4
    local dateTime=$(echo $5 | xargs)
    local verbose=$6
    local dryrun=$7

    local s3Path=${s3#s3://}
    local bucket=${s3Path%%/*}
    local path=${s3Path#*/}

    if [[ "${s3}" == "${s3Path}" ]]; then
        echo "Error: destination ${s3} is not correct"
        echo "An s3 url looks like this:  s3://<bucket>/<object>"
        exit 1
    fi

    if [[ "${dateTime}" == "" ]]; then
        dateTime=$(date -u +"%Y%m%dT%H%M%SZ")
    fi

    local url=https://s3.amazonaws.com/${bucket}/${path}
    declare -a headers

    verboseTmp=''
    if [[ "${verbose}" != "" ]]; then
        verboseTmp="/tmp/$$.get.tmp"
    fi

    local signature=$(createSignature GET "${url}" "${accessKey}" "${secretKey}" ${dateTime} 3600 's3' 'us-east-1' 1 "${verboseTmp}" headers[@])

    if [[ "${verbose}" != "" ]]; then
        cat ${verboseTmp} > ${verbose}
        cat - <<IntermediateValuese > ${verbose}

S3:             ${s3}
File:           ${file}
Prt:            ${s3Path}
Bucket:         ${bucket}
Path:           ${path}

dateTime:       ${dateTime}

URL:            ${url}
signature:      ${signature}

IntermediateValuese
    local id=$$
    if [[ ${dryrun} == 1 ]]; then
        id=dryrun
    fi
    local response="/tmp/${id}.response"
    echo                >> ${verbose}
    echo "DOWNLOAD"     >> ${verbose}
    echo "========"     >> ${verbose}
    echo "curl -o "${file}" --dump-header ${response}.header --header \"${signature}\" --header \"X-Amz-Date: ${dateTime}\" --header \"X-Amz-Content-SHA256: UNSIGNED-PAYLOAD\" ${url}" >> ${verbose}
    fi

    if [[ ${dryrun} == 0 ]]; then
        curl -o ${file} --dump-header ${response}.header --header "${signature}" --header "X-Amz-Date: ${dateTime}" --header "X-Amz-Content-SHA256: UNSIGNED-PAYLOAD" ${url}
        if [[ "${verbose}" != "" ]]; then
            echo "HEADER"           >> ${verbose}
            echo "======"           >> ${verbose}
            cat ${response}.header  >> ${verbose}
        fi
    fi
}
\$\endgroup\$
  • \$\begingroup\$ Just wondering, what is the reason behind eval file=${file}? \$\endgroup\$ – hjpotter92 Jan 16 '18 at 20:40
  • \$\begingroup\$ @hjpotter92 Can't remember. Get rid of symbolic links? \$\endgroup\$ – Martin York Jan 16 '18 at 21:00
  • \$\begingroup\$ Use local file=$(readlink -e $2) instead. :) Similarly, no need for xargs pipe in $(echo $5 | xargs) \$\endgroup\$ – hjpotter92 Jan 16 '18 at 22:27
  • \$\begingroup\$ @hjpotter92 The echo through xargs removes leading and trailing spaces in a variable. \$\endgroup\$ – Martin York Jan 16 '18 at 22:28
  • \$\begingroup\$ @hjpotter92 The use of readlink only resolves one level of indirection. It does not cope with links to links to links etc. \$\endgroup\$ – Martin York Jan 16 '18 at 22:29

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.