BucketScan.js

// Work in progress : il s'agit de la fonction permettant de collecter l'ensemble des fichiers images présents
// sur un bucket (voir si on doit mettre des limites)
// et d'envoyer leurs identifiants à la fonction de transformation d'image
// Elle doit être déployer de préférence avec un CRON


const AWS = require('aws-sdk')

// get info from env variable

const SOURCE_BUCKET = process.env.SOURCE_BUCKET
const S3_ENDPOINT_URL= process.env.S3_ENDPOINT_URL
const ACCESS_KEY = process.env.ACCESS_KEY_ID
const SECRET_KEY = process.env.ACCESS_KEY
const NEW_WIDTH = process.env.NEW_WIDTH

// Create S3 service object
const s3 = new AWS.S3({
   endpoint: S3_ENDPOINT_URL,
   credentials: {
       accessKeyId: ACCESS_KEY,
       secretAccessKey: SECRET_KEY,
   }
})

 //configure parameters for the listObjectsV2 method
const params = {
    Bucket: SOURCE_BUCKET
  }

exports.handle = async (event, context, callback) => {

  s3.listObjectsV2(params, function(err, data) {
    if (err) console.log(err, err.stack) // an error occurred
    else     console.log(data)           // successful response
  })

    // Infer the image type from the file suffix.
  const typeMatch = srcKey.match(/\.([^.]*)$/)
  if (!typeMatch) {
      console.log("Could not determine the image type.")
      return
  }
  // Check that the image type is supported
  const imageType = typeMatch[1].toLowerCase()
  if (imageType !== "jpg" && imageType !== "png") {
      console.log(`Unsupported image type: ${imageType}`)
      return
  }
  // Download the image from the S3 source bucket.

}

ImageTransform.js

// to Review : il s'agit de la fonction permettant de redimensionner une image à partir d'un bucket fourni.
// L'image source est fournie via un query parameters.
// Le bucket source est fourni via les variables d'environnement
// le bucket de destination est inscrit en dur mais cela peut être modifier.
// le cas d'usage est adapté d'un tutoriel de la doc AWS

// dependencies
const AWS = require('aws-sdk')
const sharp = require('sharp')

// get reference to S3 client
const SOURCE_BUCKET = process.env.SOURCE_BUCKET
const S3_ENDPOINT_URL= process.env.S3_ENDPOINT_URL
const ACCESS_KEY = process.env.ACCESS_KEY
const SECRET_KEY = process.env.SECRET_KEY

// Create S3 service object
const s3 = new AWS.S3({
   endpoint: S3_ENDPOINT_URL,
   credentials: {
       accessKeyId: ACCESS_KEY,
       secretAccessKey: SECRET_KEY,
   }
})

exports.handle = async (event, context, callback) => {
  // Read options from the event parameter.
  console.log("Reading options from event")
  const srcBucket = SOURCE_BUCKET
  // Object key may have spaces or unicode non-ASCII characters.
  const srcKey    = event.queryStringParameters.key
  console.log(srcKey)
  const dstBucket = `${srcBucket}-resized"`
  const dstKey    = "resized-" + srcKey
  // Infer the image type from the file suffix.
  const typeMatch = srcKey.match(/\.([^.]*)$/)
  if (!typeMatch) {
      console.log("Could not determine the image type.")
      return
  }
  // Check that the image type is supported
  const imageType = typeMatch[1].toLowerCase()
  if (imageType !== "jpg" && imageType !== "png") {
      console.log(`Unsupported image type: ${imageType}`)
      return
  }
  // Download the image from the S3 source bucket.
  try {
      const params = {
          Bucket: srcBucket,
          Key: srcKey
      }
      var origimage = await s3.getObject(params).promise()
  } catch (error) {
      console.log(error)
      return
  }
  // set thumbnail width. Resize will set the height automatically to maintain aspect ratio.
  const width  = 200
  // Use the sharp module to resize the image and save in a buffer.
  try {
      var buffer = await sharp(origimage.Body).resize(width).toBuffer()
  } catch (error) {
      console.log(error)
      return
  }
  // Upload the thumbnail image to the destination bucket
  try {
      const destparams = {
          Bucket: dstBucket,
          Key: dstKey,
          Body: buffer,
          ContentType: "image"
      };
      const putResult = await s3.putObject(destparams).promise();
  } catch (error) {
      console.log(error);
      return;
  }
  console.log('Successfully resized ' + srcBucket + '/' + srcKey +
      ' and uploaded to ' + dstBucket + '/' + dstKey);
};

reveal.js

exports.handle = async (event, context, callback) => {
    let data = {
        "event": event,
        "context": context,
        "callback": callback
    }
    console.log(data)
    return JSON.stringify(
        data
    )
}

serverless.yml

service: scaleway-workshop
configValidationMode: off

provider:
  name: scaleway
  runtime: node14
  # Global Environment variables - used in every functions
  env:
    ACCESS_KEY: ${env:SCW_SECRET_KEY}
    ACCESS_KEY_ID: ${env:SCW_ACCESS_KEY}
    SOURCE_BUCKET: serverless-demo
    DESTINATION_BUCKET: serverless-demo
    S3_ENDPOINT_URL: https://s3.fr-par.scw.cloud
    TRANSFORM_URL:  https://scalewayworkshoprug48waa.functions.fnc.fr-par.scw.cloud
  scwToken: ${env:SCW_SECRET_KEY}
  scwProject: ${env:SCW_DEFAULT_PROJECT_ID}

plugins:
  - serverless-scaleway-functions

package:
  patterns:
    - '!.gitignore'
    - '!.git/**'

functions:
  uploadfile4form:
    handler: UploadFileForm.handle
    memoryLimit: 1024
    minScale: 0
  imagetransform:
    handler: ImageTransform.handle
    memoryLimit: 1024
    minScale: 1
  uploadfile:
    handler: UploadFile64.handle
    memoryLimit: 1024
    minScale: 0
  bucketscan:
    handler: BucketScan.handle
    memoryLimit: 1024
    minScale: 1
  reveal:
    handler: reveal.handle
    minScale: 1

upload.http

POST https://scalewayworkshoprug48waa-uploadfile4form.functions.fnc.fr-par.scw.cloud
Filename: me.jpeg
Content-Type: multipart/form-data; boundary=WebAppBoundary

--WebAppBoundary
Content-Disposition: form-data; name="data"; filename="me.jpeg"

< ./me.jpeg
--WebAppBoundary--

###

upload_base64.http

POST  https://scalewayworkshoprug48waa-uploadfile.functions.fnc.fr-par.scw.cloud
Filename: me.jpeg
Content-Type: application/x-www-form-urlencoded

< me.base64.txt

###

UploadFile64.js

// To Review : c'est une fonction qui permet d'uploader une image sur un S3 (sans réveler d'information sur celui-ci)
// L'image doit être envoyée encodée en base 64
// ** base64 /Users/alex/Downloads/cat.png | curl -H "Filename: cat.png" -d @- localhost:8080 **
// Attention une fois déployé, il y a une limite sur le payload de l'appel (6MB), il n'est donc pas possible d'envoyer de grosses images
// Une alternative serait de générer une addresse temporaire pour uploader le fichier sur le S3

// dependencies
const AWS = require('aws-sdk')
const http = require('http')
// get reference to S3 client

const SOURCE_BUCKET = process.env.SOURCE_BUCKET
const S3_ENDPOINT_URL= process.env.S3_ENDPOINT_URL
const ACCESS_KEY = process.env.ACCESS_KEY
const SECRET_KEY = process.env.SECRET_KEY
const TRANSFORM_URL = process.env.TRANSFORM_URL

// Create S3 service object
const s3 = new AWS.S3({
   endpoint: S3_ENDPOINT_URL,
   credentials: {
       accessKeyId: ACCESS_KEY,
       secretAccessKey: SECRET_KEY,
   }
})

exports.handle = async (event, context, callback) => {
  // Read options from the event parameter.
  console.log("Reading options from call")
  const srcBucket = SOURCE_BUCKET
  // Object key may have spaces or unicode non-ASCII characters.
  const srcKey    = event.headers.Filename
  console.log(srcKey)
  const image64 = event.body
  // Infer the image type from the file suffix.
  const typeMatch = srcKey.match(/\.([^.]*)$/)
  if (!typeMatch) {
      console.log("Could not determine the image type.")
      return
  }
  // Check that the image type is supported
  const imageType = typeMatch[1].toLowerCase()
  if (imageType !== "jpg" && imageType !== "png") {
      console.log(`Unsupported image type: ${imageType}`)
      return {
        statusCode: 504,
        body: JSON.stringify({
            status: "fail",
            message: "Image type is not supported please send a jpg or png file check your logs for further information"
        }),
        headers: {
          "Content-Type": "application/json",
        }
      }
  }
  try {
    var buffer = Buffer.from(image64, "base64")
    } catch (error) {
        console.log(error)
        return {
            statusCode: 504,
            body: JSON.stringify({
                status: "fail",
                message: "Image could not be processed check your logs for further information"
            }),
            headers: {
              "Content-Type": "application/json",
            }
          }
    }

  // Upload the thumbnail image to the destination bucket
  try {
      const srcparams = {
          Bucket: srcBucket,
          Key: srcKey,
          Body: buffer,
          ContentType: "image"
      }
      const putResult = await s3.putObject(srcparams).promise()
  } catch (error) {
      console.log(error)
      return {
        statusCode: 504,
        body: JSON.stringify({
            status: "fail",
            message: "Image : "+srcKey+" could not be pushed to the bucket "+srcBucket+" please check your logs for more information"
        }),
        headers: {
          "Content-Type": "application/json",
        }
      }
  }
  try {
    console.log(TRANSFORM_URL+'?key='+srcKey)
    http.get(TRANSFORM_URL+'?key='+srcKey);
    } catch (error) {
        console.log(error);
        return {
            statusCode: 504,
            body: JSON.stringify({
                status: "fail",
                message: "Image : "+srcKey+" has successfully been pushed to the bucket "+srcBucket+ "but the transformation call failed please check your logs"
            }),
            headers: {
              "Content-Type": "application/json",
            }
          }
    }
  console.log('Successfully upload ' + srcBucket + '/' + srcKey);
  return {
    statusCode: 201,
    body: JSON.stringify({
        status: "ok",
        message: "Image : "+srcKey+" has successfully been pushed to the bucket "+srcBucket+ "and the transform request has been performed"
    }),
    headers: {
      "Content-Type": "application/json",
    }
  }
};

UploadFileForm.js

// To Review : c'est une fonction qui permet d'uploader une image sur un S3 (sans réveler d'information sur celui-ci)
// L'image doit être envoyée au travers d'un multipart-form, une librairie dédiée est utilisée
// **curl -i -X POST \
//    -H "Content-Type: multipart/form-data" \
//    -H "Filename: hibou2.jpg"\
//    -F "[email protected]" \
//  'https://fileuploadva0yjbmf-nodeupload.functions.fnc.fr-par.scw.cloud'**
// Attention une fois déployé, il y a une limite sur le payload de l'appel (6MB), il n'est donc pas possible d'envoyer de grosses images
// Une alternative serait de générer une addresse temporaire pour uploader le fichier sur le S3

// dependencies
const AWS = require('aws-sdk')
const parser = require('lambda-multipart-parser')
// get reference to S3 client
const srcBucket = process.env.SOURCE_BUCKET
const S3_ENDPOINT_URL = process.env.S3_ENDPOINT_URL
const ACCESS_KEY = process.env.ACCESS_KEY
const SECRET_KEY = process.env.SECRET_KEY

// Create S3 service object
const s3 = new AWS.S3({
    endpoint: S3_ENDPOINT_URL,
    credentials: {
        accessKeyId: ACCESS_KEY,
        secretAccessKey: SECRET_KEY,
    }
})

exports.handle = async (event, context, callback) => {
    //Parsing event content
    const result = await parser.parse(event)
    const file = result.files[0]
    let buffer = file.content
    let srcKey = file.filename

    try {
        let imageType = file.contentType
        // Check that the image type is supported
        if (imageType !== "image/jpg" && imageType !== "image/png") {
            console.log(`Unsupported image type: ${imageType}`)
            return {
                statusCode: 504,
                body: JSON.stringify({
                    status: "fail",
                    message: "Image type is not supported please send a jpg or png file check your logs for further information"
                }),
                headers: {
                    "Content-Type": "application/json",
                }
            }
        }
    } catch (error) {
        console.log(error)
        return {
            statusCode: 504,
            body: JSON.stringify({
                status: "fail",
                message: "Image could not be processed check your logs for further information"
            }),
            headers: {
                "Content-Type": "application/json",
            }
        }
    }

    // Upload the thumbnail image to the destination bucket
    try {
        const srcparams = {
            Bucket: srcBucket,
            Key: srcKey,
            Body: buffer,
            ContentType: "image"
        }
        console.log("updating to s3")
        const putResult = await s3.putObject(srcparams).promise()
    } catch (error) {
        console.log(error)
        return {
            statusCode: 504,
            body: JSON.stringify({
                status: "fail",
                message: "Image : " + srcKey + " could not be pushed to the bucket " + srcBucket + " please check your logs for more information"
            }),
            headers: {
                "Content-Type": "application/json",
            }
        }
    }
    try {
        console.log(TRANSFORM_URL + '?key=' + srcKey)
        // We trigger an event
        http.get(TRANSFORM_URL + '?key=' + srcKey)
    } catch (error) {
        console.log(error)
        return {
            statusCode: 504,
            body: JSON.stringify({
                status: "fail",
                message: "Image : " + srcKey + " has successfully been pushed to the bucket " + srcBucket + "but the transformation call failed please check your logs"
            }),
            headers: {
                "Content-Type": "application/json",
            }
        }
    }
    console.log('Successfully upload ' + srcBucket + '/' + srcKey)
    return {
        statusCode: 201,
        body: JSON.stringify({
            status: "ok",
            message: "Image : " + srcKey + " has successfully been pushed to the bucket " + srcBucket + " and the transform request has been performed"
        }),
        headers: {
            "Content-Type": "application/json",
        }
    }
}