From 0d2a0d269cec36bd365ab131e564e23ec7902dde Mon Sep 17 00:00:00 2001 From: manuel Date: Fri, 27 Apr 2018 12:00:03 +0100 Subject: [PATCH] leading slash in object key does not alter listBucket --- .../s3mock/provider/FileProvider.scala | 39 +++++++++++-------- .../s3mock/provider/InMemoryProvider.scala | 7 +++- .../io/findify/s3mock/ListBucketTest.scala | 17 ++++++++ 3 files changed, 46 insertions(+), 17 deletions(-) diff --git a/src/main/scala/io/findify/s3mock/provider/FileProvider.scala b/src/main/scala/io/findify/s3mock/provider/FileProvider.scala index 99c67a2..0784416 100644 --- a/src/main/scala/io/findify/s3mock/provider/FileProvider.scala +++ b/src/main/scala/io/findify/s3mock/provider/FileProvider.scala @@ -1,4 +1,5 @@ package io.findify.s3mock.provider + import java.util.UUID import java.io.{FileInputStream, File => JFile} @@ -18,7 +19,7 @@ import scala.util.Random /** * Created by shutty on 8/9/16. */ -class FileProvider(dir:String) extends Provider with LazyLogging { +class FileProvider(dir: String) extends Provider with LazyLogging { val workDir = File(dir) if (!workDir.exists) workDir.createDirectories() @@ -39,18 +40,21 @@ class FileProvider(dir:String) extends Provider with LazyLogging { case pos => Some(p + dir.substring(p.length, pos) + d) } } + val prefixNoLeadingSlash = prefix.getOrElse("").dropWhile(_ == '/') val bucketFile = File(s"$dir/$bucket/") if (!bucketFile.exists) throw NoSuchBucketException(bucket) val bucketFileString = fromOs(bucketFile.toString) val bucketFiles = bucketFile.listRecursively.filter(f => { - val fString = fromOs(f.toString).drop(bucketFileString.length).dropWhile(_ == '/') - fString.startsWith(prefixNoLeadingSlash) && !f.isDirectory - }) + val fString = fromOs(f.toString).drop(bucketFileString.length).dropWhile(_ == '/') + + if (fString.startsWith("%2F")) fString.drop(3).startsWith(prefixNoLeadingSlash) && !f.isDirectory + else fString.startsWith(prefixNoLeadingSlash) && !f.isDirectory + }) val files = bucketFiles.map(f => { val stream = new FileInputStream(f.toJava) val md5 = DigestUtils.md5Hex(stream) - Content(fromOs(f.toString).drop(bucketFileString.length+1).dropWhile(_ == '/'), DateTime(f.lastModifiedTime.toEpochMilli), md5, f.size, "STANDARD") + Content(fromOs(f.toString).drop(bucketFileString.length + 1).dropWhile(_ == '/'), DateTime(f.lastModifiedTime.toEpochMilli), md5, f.size, "STANDARD") }).toList logger.debug(s"listing bucket contents: ${files.map(_.key)}") val commonPrefixes = delimiter match { @@ -60,16 +64,17 @@ class FileProvider(dir:String) extends Provider with LazyLogging { val filteredFiles = files.filterNot(f => commonPrefixes.exists(p => f.key.startsWith(p))) val count = maxkeys.getOrElse(Int.MaxValue) val result = filteredFiles.sortBy(_.key) - ListBucket(bucket, prefix, delimiter, commonPrefixes, result.take(count), isTruncated = result.size>count) + ListBucket(bucket, prefix, delimiter, commonPrefixes, result.take(count), isTruncated = result.size > count) } - override def createBucket(name:String, bucketConfig:CreateBucketConfiguration) = { + override def createBucket(name: String, bucketConfig: CreateBucketConfiguration) = { val bucket = File(s"$dir/$name") if (!bucket.exists) bucket.createDirectory() logger.debug(s"creating bucket $name") CreateBucket(name) } - override def putObject(bucket:String, key:String, data:Array[Byte], objectMetadata: ObjectMetadata): Unit = { + + override def putObject(bucket: String, key: String, data: Array[Byte], objectMetadata: ObjectMetadata): Unit = { val bucketFile = File(s"$dir/$bucket") val file = File(s"$dir/$bucket/$key") if (!bucketFile.exists) throw NoSuchBucketException(bucket) @@ -79,7 +84,8 @@ class FileProvider(dir:String) extends Provider with LazyLogging { objectMetadata.setLastModified(org.joda.time.DateTime.now().toDate) metadataStore.put(bucket, key, objectMetadata) } - override def getObject(bucket:String, key:String): GetObjectData = { + + override def getObject(bucket: String, key: String): GetObjectData = { val bucketFile = File(s"$dir/$bucket") val file = File(s"$dir/$bucket/$key") logger.debug(s"reading object for s://$bucket/$key") @@ -90,7 +96,7 @@ class FileProvider(dir:String) extends Provider with LazyLogging { GetObjectData(file.byteArray, meta) } - override def putObjectMultipartStart(bucket:String, key:String, metadata: ObjectMetadata):InitiateMultipartUploadResult = { + override def putObjectMultipartStart(bucket: String, key: String, metadata: ObjectMetadata): InitiateMultipartUploadResult = { val id = Math.abs(Random.nextLong()).toString val bucketFile = File(s"$dir/$bucket") if (!bucketFile.exists) throw NoSuchBucketException(bucket) @@ -99,7 +105,8 @@ class FileProvider(dir:String) extends Provider with LazyLogging { logger.debug(s"starting multipart upload for s3://$bucket/$key") InitiateMultipartUploadResult(bucket, key, id) } - override def putObjectMultipartPart(bucket:String, key:String, partNumber:Int, uploadId:String, data:Array[Byte]) = { + + override def putObjectMultipartPart(bucket: String, key: String, partNumber: Int, uploadId: String, data: Array[Byte]) = { val bucketFile = File(s"$dir/$bucket") if (!bucketFile.exists) throw NoSuchBucketException(bucket) val file = File(s"$dir/.mp/$bucket/$key/$uploadId/$partNumber") @@ -107,7 +114,7 @@ class FileProvider(dir:String) extends Provider with LazyLogging { file.writeByteArray(data)(OpenOptions.default) } - override def putObjectMultipartComplete(bucket:String, key:String, uploadId:String, request:CompleteMultipartUpload): CompleteMultipartUploadResult = { + override def putObjectMultipartComplete(bucket: String, key: String, uploadId: String, request: CompleteMultipartUpload): CompleteMultipartUploadResult = { val bucketFile = File(s"$dir/$bucket") if (!bucketFile.exists) throw NoSuchBucketException(bucket) val files = request.parts.map(part => File(s"$dir/.mp/$bucket/$key/$uploadId/${part.partNumber}")) @@ -118,7 +125,7 @@ class FileProvider(dir:String) extends Provider with LazyLogging { file.writeBytes(data.toIterator) File(s"$dir/.mp/$bucket/$key").delete() val hash = file.md5 - metadataStore.get(bucket, key).foreach {m => + metadataStore.get(bucket, key).foreach { m => m.setContentMD5(hash) m.setLastModified(org.joda.time.DateTime.now().toDate) } @@ -142,13 +149,13 @@ class FileProvider(dir:String) extends Provider with LazyLogging { } - override def copyObjectMultipart(sourceBucket: String, sourceKey: String, destBucket: String, destKey: String, part: Int, uploadId:String, fromByte: Int, toByte: Int, newMeta: Option[ObjectMetadata] = None): CopyObjectResult = { + override def copyObjectMultipart(sourceBucket: String, sourceKey: String, destBucket: String, destKey: String, part: Int, uploadId: String, fromByte: Int, toByte: Int, newMeta: Option[ObjectMetadata] = None): CopyObjectResult = { val data = getObject(sourceBucket, sourceKey).bytes.slice(fromByte, toByte + 1) putObjectMultipartPart(destBucket, destKey, part, uploadId, data) new CopyObjectResult(DateTime.now, DigestUtils.md5Hex(data)) } - override def deleteObject(bucket:String, key:String): Unit = { + override def deleteObject(bucket: String, key: String): Unit = { val file = File(s"$dir/$bucket/$key") logger.debug(s"deleting object s://$bucket/$key") if (!file.exists) throw NoSuchKeyException(bucket, key) @@ -158,7 +165,7 @@ class FileProvider(dir:String) extends Provider with LazyLogging { } } - override def deleteBucket(bucket:String): Unit = { + override def deleteBucket(bucket: String): Unit = { val bucketFile = File(s"$dir/$bucket") logger.debug(s"deleting bucket s://$bucket") if (!bucketFile.exists) throw NoSuchBucketException(bucket) diff --git a/src/main/scala/io/findify/s3mock/provider/InMemoryProvider.scala b/src/main/scala/io/findify/s3mock/provider/InMemoryProvider.scala index a355802..aad2a59 100644 --- a/src/main/scala/io/findify/s3mock/provider/InMemoryProvider.scala +++ b/src/main/scala/io/findify/s3mock/provider/InMemoryProvider.scala @@ -45,10 +45,15 @@ class InMemoryProvider extends Provider with LazyLogging { } } + def leadingSlash(prefix: String)(key: String): Boolean = { + if(key.startsWith("%2F")) ("/" + key.drop(3)).startsWith(prefix) + else key.startsWith(prefix) + } + val prefix2 = prefix.getOrElse("") bucketDataStore.get(bucket) match { case Some(bucketContent) => - val matchingKeys = bucketContent.keysInBucket.filterKeys(_.startsWith(prefix2)) + val matchingKeys = bucketContent.keysInBucket.filterKeys(leadingSlash(prefix2)) val matchResults = matchingKeys map { case (name, content) => Content(name, content.lastModificationTime, DigestUtils.md5Hex(content.data), content.data.length, "STANDARD") } diff --git a/src/test/scala/io/findify/s3mock/ListBucketTest.scala b/src/test/scala/io/findify/s3mock/ListBucketTest.scala index c7cf2f9..ac40f4f 100644 --- a/src/test/scala/io/findify/s3mock/ListBucketTest.scala +++ b/src/test/scala/io/findify/s3mock/ListBucketTest.scala @@ -204,5 +204,22 @@ class ListBucketTest extends S3MockTest { val list = s3.listObjects("list10").getObjectSummaries.asScala.toList list.find(_.getKey == "foo").map(_.getLastModified.after(DateTime.now().minusMinutes(1).toDate)) shouldBe Some(true) } + + it should "return non empty list if prefix is correct for objects with leading slash" in { + s3.createBucket("test") + s3.putObject("test", "/one/foo1", "xxx") + s3.putObject("test", "/one/foo2", "xxx") + s3.putObject("test", "/one/foo3", "xxx") + s3.getObject("test", "/one/foo1") + s3.listObjects("test", "/one").getObjectSummaries.asScala.isEmpty shouldBe false + } + + it should "return empty list if prefix is incorrect for objects with leading slash" in { + s3.createBucket("test") + s3.putObject("test", "/one/foo1", "xxx") + s3.putObject("test", "/one/foo2", "xxx") + s3.putObject("test", "/one/foo3", "xxx") + s3.listObjects("test", "%2Fone").getObjectSummaries.asScala.isEmpty shouldBe true + } } }