From 64bcb5a4fabdc08363427715685832e72fc0def0 Mon Sep 17 00:00:00 2001 From: borkupe Date: Sat, 8 Jul 2023 12:59:22 +0200 Subject: [PATCH 1/2] upgrade basic-s3-thumbnail example + refactor test I noticed that I am testing thumbnailer module when I compared the generated thumbnail with a stored one. This can lead test failures. So I mocked the thumbnail generation. --- examples/basic-s3-thumbnail/Cargo.toml | 11 +++-- examples/basic-s3-thumbnail/README.md | 16 +++++++ examples/basic-s3-thumbnail/src/main.rs | 44 +++++++++--------- examples/basic-s3-thumbnail/src/s3.rs | 4 +- .../basic-s3-thumbnail/testdata/image.png | Bin 282 -> 0 bytes .../basic-s3-thumbnail/testdata/thumbnail.png | Bin 82 -> 0 bytes 6 files changed, 46 insertions(+), 29 deletions(-) delete mode 100644 examples/basic-s3-thumbnail/testdata/image.png delete mode 100644 examples/basic-s3-thumbnail/testdata/thumbnail.png diff --git a/examples/basic-s3-thumbnail/Cargo.toml b/examples/basic-s3-thumbnail/Cargo.toml index dc5e67f8..6bbe11b7 100644 --- a/examples/basic-s3-thumbnail/Cargo.toml +++ b/examples/basic-s3-thumbnail/Cargo.toml @@ -21,13 +21,14 @@ serde = "1" tokio = { version = "1", features = ["macros"] } tracing = { version = "0.1" } tracing-subscriber = { version = "0.3", default-features = false, features = ["ansi", "fmt"] } -aws-config = "0.54.1" -aws-sdk-s3 = "0.24.0" +aws-config = "0.55" +aws-smithy-http = "0.55.3" +aws-sdk-s3 = "0.28" thumbnailer = "0.4.0" mime = "0.3.16" -async-trait = "0.1.66" +async-trait = "0.1.68" [dev-dependencies] -mockall = "0.11.3" -tokio-test = "0.4.2" +mockall = "0.11" +tokio-test = "0.4" chrono = "0.4" diff --git a/examples/basic-s3-thumbnail/README.md b/examples/basic-s3-thumbnail/README.md index de2d56f8..000874cc 100644 --- a/examples/basic-s3-thumbnail/README.md +++ b/examples/basic-s3-thumbnail/README.md @@ -5,6 +5,22 @@ it downloads the created file, generates a thumbnail from it (it assumes that the file is an image) and uploads it to S3 into a bucket named [original-bucket-name]-thumbs. +## Set up +1. Create a lambda function and upload the bootloader.zip +2. Go to aws services S3 +3. Create a bucket, let's say with name bucketx +4. Create another bucket bucketx-thumbs +5. Got to the bucketx properties tab, event notifications +6. Create lambda event notification for "all object create event" and select your lambda function +7. Go to the lambda function, configuration and open the role name +8. Add "AmazonS3FullAccess" permission + +## Test + +1. Go to S3 and upload a png picture into bucketx. Beware to not have spaces or any special characters in the file name +2. Go to S3 bucketx-thumbs and check if an image is created there. + + ## Build & Deploy 1. Install [cargo-lambda](https://github.com/cargo-lambda/cargo-lambda#installation) diff --git a/examples/basic-s3-thumbnail/src/main.rs b/examples/basic-s3-thumbnail/src/main.rs index f9cb0716..ad8b3110 100644 --- a/examples/basic-s3-thumbnail/src/main.rs +++ b/examples/basic-s3-thumbnail/src/main.rs @@ -1,10 +1,7 @@ -use std::io::Cursor; - use aws_lambda_events::{event::s3::S3Event, s3::S3EventRecord}; use aws_sdk_s3::Client as S3Client; use lambda_runtime::{run, service_fn, Error, LambdaEvent}; use s3::{GetFile, PutFile}; -use thumbnailer::{create_thumbnails, ThumbnailSize}; mod s3; @@ -86,7 +83,12 @@ fn get_file_props(record: S3EventRecord) -> Result<(String, String), String> { Ok((bucket, key)) } +#[cfg(not(test))] fn get_thumbnail(vec: Vec, size: u32) -> Result, String> { + use std::io::Cursor; + + use thumbnailer::{create_thumbnails, ThumbnailSize}; + let reader = Cursor::new(vec); let mime = mime::IMAGE_PNG; let sizes = [ThumbnailSize::Custom((size, size))]; @@ -129,12 +131,19 @@ async fn main() -> Result<(), Error> { Ok(()) } +#[cfg(test)] +fn get_thumbnail(vec: Vec, _size: u32) -> Result, String> { + let s = unsafe { std::str::from_utf8_unchecked(&vec) }; + + match s { + "IMAGE" => Ok("THUMBNAIL".into()), + _ => Err("Input is not IMAGE".to_string()), + } +} + #[cfg(test)] mod tests { use std::collections::HashMap; - use std::fs::File; - use std::io::BufReader; - use std::io::Read; use super::*; use async_trait::async_trait; @@ -144,7 +153,7 @@ mod tests { use aws_lambda_events::s3::S3Object; use aws_lambda_events::s3::S3RequestParameters; use aws_lambda_events::s3::S3UserIdentity; - use aws_sdk_s3::error::GetObjectError; + use aws_sdk_s3::operation::get_object::GetObjectError; use lambda_runtime::{Context, LambdaEvent}; use mockall::mock; use s3::GetFile; @@ -174,15 +183,14 @@ mod tests { let mut mock = MockFakeS3Client::new(); mock.expect_get_file() - .withf(|b: &str, k: &str| b.eq(bucket) && k.eq(key)) - .returning(|_1, _2| Ok(get_file("testdata/image.png"))); + .withf(|b, k| b.eq(bucket) && k.eq(key)) + .returning(|_1, _2| Ok("IMAGE".into())); mock.expect_put_file() - .withf(|bu: &str, ke: &str, by| { - let thumbnail = get_file("testdata/thumbnail.png"); - return bu.eq("test-bucket-thumbs") && ke.eq(key) && by == &thumbnail; + .withf(|bu, ke, by| { + return bu.eq("test-bucket-thumbs") && ke.eq(key) && by.eq("THUMBNAIL".as_bytes()); }) - .returning(|_1, _2, _3| Ok("Done".to_string())); + .return_const(Ok("Done".to_string())); let payload = get_s3_event("ObjectCreated", bucket, key); let event = LambdaEvent { payload, context }; @@ -192,16 +200,6 @@ mod tests { assert_eq!((), result); } - fn get_file(name: &str) -> Vec { - let f = File::open(name); - let mut reader = BufReader::new(f.unwrap()); - let mut buffer = Vec::new(); - - reader.read_to_end(&mut buffer).unwrap(); - - return buffer; - } - fn get_s3_event(event_name: &str, bucket_name: &str, object_key: &str) -> S3Event { return S3Event { records: (vec![get_s3_event_record(event_name, bucket_name, object_key)]), diff --git a/examples/basic-s3-thumbnail/src/s3.rs b/examples/basic-s3-thumbnail/src/s3.rs index 83ef7bc7..17d7f975 100644 --- a/examples/basic-s3-thumbnail/src/s3.rs +++ b/examples/basic-s3-thumbnail/src/s3.rs @@ -1,5 +1,7 @@ use async_trait::async_trait; -use aws_sdk_s3::{error::GetObjectError, types::ByteStream, Client as S3Client}; +use aws_sdk_s3::operation::get_object::GetObjectError; +use aws_sdk_s3::Client as S3Client; +use aws_smithy_http::byte_stream::ByteStream; #[async_trait] pub trait GetFile { diff --git a/examples/basic-s3-thumbnail/testdata/image.png b/examples/basic-s3-thumbnail/testdata/image.png deleted file mode 100644 index 078d155f6bf6735eb087eb0195b3e35f9f424d04..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 282 zcmeAS@N?(olHy`uVBq!ia0vp^DIm-UBp4!QuJ{S0SkfJR9T^xl_H+M9WCijSl0AZa z85pY67#JE_7#My5g&JNkFq9fFFuY1&V6d9Oz#v{QXIG#NP=cu>$S;_Ip=|P53lJ~K z+uenM@oty!5+IMg#M9T6{W-Ikn3DL(-uF5{ArVg(#}JFt$q5pyixWh8ngSjC85meA z7#KARcm4s&tCqM%l%ynf4NqV|ChEy=Vy|59;VQAR!XXWJ= d863$M85tR8F)&*H Date: Sun, 9 Jul 2023 12:26:41 +0200 Subject: [PATCH 2/2] refactor test for basic-s3-object-lambda example --- .../src/main.rs | 61 ++++++++----------- 1 file changed, 24 insertions(+), 37 deletions(-) diff --git a/examples/basic-s3-object-lambda-thumbnail/src/main.rs b/examples/basic-s3-object-lambda-thumbnail/src/main.rs index 7786f56e..27186fa0 100644 --- a/examples/basic-s3-object-lambda-thumbnail/src/main.rs +++ b/examples/basic-s3-object-lambda-thumbnail/src/main.rs @@ -1,10 +1,9 @@ -use std::{error, io::Cursor}; +use std::error; use aws_lambda_events::s3::object_lambda::{GetObjectContext, S3ObjectLambdaEvent}; use aws_sdk_s3::Client as S3Client; use lambda_runtime::{run, service_fn, Error, LambdaEvent}; use s3::{GetFile, SendFile}; -use thumbnailer::{create_thumbnails, ThumbnailSize}; mod s3; @@ -35,28 +34,21 @@ pub(crate) async fn function_handler( let thumbnail = get_thumbnail(image, size); tracing::info!("thumbnail created. Length: {}", thumbnail.len()); - // It sends the thumbnail back to the user - client.send_file(route, token, thumbnail).await - - /* - match client.send_file(route, token, thumbnail).await { - Ok(msg) => tracing::info!(msg), - Err(msg) => tracing::info!(msg) - }; - - tracing::info!("handler ends"); - - Ok(()) - */ } +#[cfg(not(test))] fn get_thumbnail(vec: Vec, size: u32) -> Vec { - let reader = Cursor::new(vec); - let mut thumbnails = create_thumbnails(reader, mime::IMAGE_PNG, [ThumbnailSize::Custom((size, size))]).unwrap(); + let reader = std::io::Cursor::new(vec); + let mut thumbnails = thumbnailer::create_thumbnails( + reader, + mime::IMAGE_PNG, + [thumbnailer::ThumbnailSize::Custom((size, size))], + ) + .unwrap(); let thumbnail = thumbnails.pop().unwrap(); - let mut buf = Cursor::new(Vec::new()); + let mut buf = std::io::Cursor::new(Vec::new()); thumbnail.write_png(&mut buf).unwrap(); buf.into_inner() @@ -88,11 +80,17 @@ async fn main() -> Result<(), Error> { } #[cfg(test)] -mod tests { - use std::fs::File; - use std::io::BufReader; - use std::io::Read; +fn get_thumbnail(vec: Vec, _size: u32) -> Vec { + let s = unsafe { std::str::from_utf8_unchecked(&vec) }; + match s { + "IMAGE" => "THUMBNAIL".into(), + _ => "Input is not IMAGE".into(), + } +} + +#[cfg(test)] +mod tests { use super::*; use async_trait::async_trait; use aws_lambda_events::s3::object_lambda::Configuration; @@ -125,13 +123,12 @@ mod tests { let mut mock = MockFakeS3Client::new(); mock.expect_get_file() - .withf(|u: &String| u.eq("S3_URL")) - .returning(|_1| Ok(get_file("testdata/image.png"))); + .withf(|u| u.eq("S3_URL")) + .returning(|_1| Ok("IMAGE".into())); mock.expect_send_file() - .withf(|r: &String, t: &String, by| { - let thumbnail = get_file("testdata/thumbnail.png"); - return r.eq("O_ROUTE") && t.eq("O_TOKEN") && by == &thumbnail; + .withf(|r, t, by| { + return r.eq("O_ROUTE") && t.eq("O_TOKEN") && by == "THUMBNAIL".as_bytes(); }) .returning(|_1, _2, _3| Ok("File sent.".to_string())); @@ -144,16 +141,6 @@ mod tests { assert_eq!(("File sent."), result); } - fn get_file(name: &str) -> Vec { - let f = File::open(name); - let mut reader = BufReader::new(f.unwrap()); - let mut buffer = Vec::new(); - - reader.read_to_end(&mut buffer).unwrap(); - - return buffer; - } - fn get_s3_event() -> S3ObjectLambdaEvent { return S3ObjectLambdaEvent { x_amz_request_id: ("ID".to_string()),